Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+evaluation-pipeline/assets/babylm.png filter=lfs diff=lfs merge=lfs -text
+evaluation-pipeline/docs/img/fewshot_example_gpt3.png filter=lfs diff=lfs merge=lfs -text
+evaluation-pipeline/sample_predictions.json filter=lfs diff=lfs merge=lfs -text

evaluation-pipeline/.coveragerc

@@ -0,0 +1,21 @@
+[run]
+omit =
+   # Requires manual data download
+   lm_eval/tasks/jigsaw_unintended_bias.py
+
+[report]
+exclude_lines =
+    # Skip any pass lines such as may be used for @abstractmethod
+    pass
+
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+    return NotImplemented

evaluation-pipeline/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E266, E501, W503, F403, F401, C901
+max-line-length = 127
+max-complexity = 10
+select = B,C,E,F,W,T4,B9

evaluation-pipeline/.gitignore

@@ -0,0 +1,56 @@
+# Distribution / Packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.pyc
+*.log
+*.db
+MANIFEST
+
+# Directories
+data/
+outputs/
+lm_cache/
+
+# Editors
+.idea
+.vscode
+.ipynb_checkpoints
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Unit test / coverage reports
+.coverage
+.coverage.*
+.cache
+
+# local testing files
+test/
+
+# baseline models
+baselines/
+
+# outputs from running scripts
+.out
+.err

evaluation-pipeline/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lm_eval/datasets/biomedical"]
+	path = lm_eval/datasets/biomedical
+	url = https://github.com/bigscience-workshop/biomedical

evaluation-pipeline/.pre-commit-config.yaml

@@ -0,0 +1,44 @@
+# Ignore test linting to avoid conflicting changes to version stability.
+exclude: ^tests/testdata/
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-byte-order-marker
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: destroyed-symlinks
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: no-commit-to-branch
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+      - id: fix-byte-order-marker
+        exclude: docs/CNAME
+      - id: fix-encoding-pragma
+        args: [--remove]
+      - id: mixed-line-ending
+        args: [--fix=lf]
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3.8
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.1.0
+    hooks:
+      - id: codespell
+        exclude: >
+          (?x)^(
+              .*\.json|ignore.txt
+          )$
+        args: [--check-filenames, --check-hidden, --ignore-words=ignore.txt]

evaluation-pipeline/CODEOWNERS

@@ -0,0 +1 @@
+* @jon-tow @leogao2 @StellaAthena

evaluation-pipeline/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 EleutherAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evaluation-pipeline/README.md

@@ -0,0 +1,237 @@
+# BabyLM Evaluation Pipeline
+![BabyLM Challenge](assets/babylm.png)
+
+## Overview
+
+This code provides the backend for the BabyLM Challenge's evaluation pipeline.
+
+We provide support for zero-shot evaluations on BLiMP, as well as scripts for fine-tuning HuggingFace-based models on GLUE and MSGS tasks.
+
+We also provide a [Colab demo](https://colab.research.google.com/drive/1HX2D3wztO81tKcqCeV_ecRcEUseBVuTc?usp=sharing) of the evaluation pipeline as a demonstration of how to use the code.
+
+If you have questions about or suggestions for this code, please open an issue and consider [joining our Slack](https://join.slack.com/t/babylmchallenge/shared_invite/zt-1s8el4mro-qvVO447l3POBZcUNvMWQcg). We also welcome pull requests!
+
+## Installation
+
+To install dependencies, run this:
+
+```bash
+git clone https://github.com/babylm/evaluation-pipeline
+cd evaluation-pipeline
+pip install -e ".[dev]"
+pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113
+```
+
+If your GPU is compatible with CUDA 10, replace all instances of `cu113` with `cu102`.
+
+### Data
+We provide versions of BLiMP, GLUE, and MSGS which have been filtered according to the vocabulary of the `strict-small` dataset. We filter for examples where each word has appeared in our training set at least twice.
+
+Unzip the dataset into the root directory of this repository: `unzip filter_data.zip`.
+
+## Usage
+### Zero-shot Evaluation
+To evaluate a model on zero-shot tasks like BLiMP and the held-out BLiMP supplement tasks:
+
+```bash
+python babylm_eval.py 'path/to/model_and_tokenizer' 'model_type'
+```
+
+Where `model_type` is one of "encoder", "decoder" or "encoder-decoder".
+
+### Fine-tuning
+To fine-tune and evaluate a model on tasks that require fine-tuning, like the (Super)GLUE tasks or held-out MSGS tasks:
+
+```bash
+./finetune_all_tasks.sh 'path/to/model_and_tokenizer'
+```
+
+#### Hyperparameters
+This script contains hyperparameter defaults that should work for a variety of model sizes, architectures, and tasks. You may adjust these hyperparameters as you wish, though we ask that you submit the best hyperparmeter settings in a README file if you don't use the defaults.
+
+Here are the defaults that we use:
+| Hyperparameter | Value |
+| -------------- | ----- |
+| Initial learning rate | 5e-5 |
+| Batch size | 64 |
+| Maximum epochs | 10 |
+| Evaluate every (steps) | 200 |
+| Patience | 10 |
+| Random seed | 12 |
+
+## Uploading Results
+We provide a shell script that will collect your results into a single file:
+
+```bash
+./collect_results.py path/to/model_and_tokenizer
+```
+
+This will output a file called `all_predictions.json` in the root folder of this repository. We will ask you to upload this file to a submission portal.
+
+We will also ask you to share a link where we can download your model and tokenizer.
+
+### Format of Predictions
+If you wish to submit your results and you are not using the `collect_results.py` script, please ensure that your predictions file conforms to the submission format (example provided here as `sample_predictions.json`). This is a file consisting of line-separated JSON objects, where each line corresponds to a single subtask.
+
+For each line, the JSON object includes a `task` field ("blimp", "glue", "supplement", or "msgs"), a `sub_task` field (the specific task, like "cola" or "anaphor_agreement"), and a `predictions` field, which is a list of JSON objects containing example IDs and predictions for those examples. Here is an example:
+
+```
+{"task": "glue", "sub_task": "mnli", "predictions": [{"id": "mnli_0", "pred": 0}, {"id": "mnli_1": "pred": 1}, ..., {"id": "mnli_6561", "pred": 1}]}
+```
+
+### Age-of-acquisition prediction Evaluation
+This evaluation is based on Portelance, Duan, Lupyan and Frank 2023 (see citation below).
+
+If you want to run it, run the zero-shot evaluation script with the "--run_aoa" flag:
+
+```bash
+python babylm_eval.py 'path/to/model_and_tokenizer' 'model_type' --run_aoa
+```
+
+Note, the evaluation requires access to forward pass labels from your tokenizer. It currently expects the tokenizer to either produce them under the key "labels" if the model type is a "decoder" where labels represent the shifted "input_ids", or if no labels are provided, it will set the "labels" to be equal to the "input_ids" (this is done automatically for "encoder" and "encoder-decoder" type models. In the event that your labels are not equal to the input_ids, please make sure your tokenizer contains them under the key "labels".
+
+Once it runs, it will produce two json files in a folder called "aoa_prediction" in the model directory provided. One of the files contains the estimated average surprisal of words for the model in child directed utterances taken from CHILDES. The other contains the results of the evaluation. Models are evaluated using leave-one-out cross validation. The results are Mean Absolute Deviation (MAD) scores in months between the actual average age-of-acquisition (AoA) of these words by American English speaking children and the predicted AoA based on the models average surprisal scores (the closer the MAD scores are to zero, the better). MAD scores are provided over all the words, over nouns, over predicates, and over function words. Previous work has found that models tend to do better at predicting the AoA of predicates and function words over nouns.
+
+The better the fit, the better a model's predictions and the actual AoA of words in kids (the smaller the MAD scores), the more the order in which models learn words resembles the order in which children tend to learn words.
+
+Note that, while we do not require you to run this evaluation or submit your score for our evaluation, we highly encourage you to compute this metric and discuss it in your paper!
+
+## Baselines
+We provide a series of baseline models that we train on our strict or strict-small dataset. These are [hosted on HuggingFace](https://huggingface.co/babylm).
+
+We simply take the hyperparameters used to pre-train the original versions of these models, and train them on our strict or strict-small datasets. While we do reduce the context length and, in some cases, the batch size, these are otherwise minimally modified.
+
+Here are baseline scores. These are all accuracies, unless otherwise noted by (F1), where we use macro-F1. Random chance accuracy on all BLiMP tasks is 50.
+
+**Strict-small Track**
+
+*BLiMP*
+| Model | Anaphor Agr. | Agr. Structure | Binding | Control/Raising | D-N Agr. | Ellipsis | Filler-Gap | Irregular Forms | Island Effects | NPI Licensing | Quantifiers | S-V Agr. |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| OPT-125m | 63.8 | 70.6 | 67.1 | 66.5 | 78.5 | 62 | 63.8 | 67.5 | 48.6 | 46.7 | 59.6 | 56.9 |
+| RoBERTa-base | 81.5 | 67.1 | 67.3 | 67.9 | 90.8 | 76.4 | 63.5 | 87.4 | 39.9 | 55.9 | 70.5 | 65.4 |
+| T5-base | 68.9 | 63.8 | 60.4 | 60.9 | 72.2 | 34.4 | 48.2 | 77.6 | 45.6 | 47.8 | 61.2 | 65.0 |
+
+*BLiMP Supplement*
+| Model | Hypernym | QA Congruence (easy) | QA Congruence (tricky) | Subj.-Aux. Inversion | Turn Taking |
+| --- | --- | --- | --- | --- | --- |
+| OPT-125m | 50.0 | 54.7 | 31.5 | 80.3 | 57.1 |
+| RoBERTa-base | 49.4 | 31.3 | 32.1 | 71.7 | 53.2 |
+| T5-base | 48.0 | 40.6 | 21.2 | 64.9 | 45.0 |
+
+*(Super)GLUE*
+| Model | CoLA | SST-2 | MRPC (F1) | QQP (F1) | MNLI | MNLI-mm | QNLI | RTE | BoolQ | MultiRC | WSC |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| *Majority label* | *69.5* | *50.2* | *82* | *53.1* | *35.7* | *35.7* | *35.4* | *53.1* | *50.5* | *59.9* | *53.2* | *61.4* |
+| OPT-125m | 64.6 | 81.9 | 72.5 | 60.4 | 57.6 | 60.0 | 61.5 | 60.0 | 63.3 | 55.2 | 60.2 |
+| RoBERTa-base | 70.8 | 87.0 | 79.2 | 73.7 | 73.2 | 74.0 | 77.0 | 61.6 | 66.3 | 61.4 | 61.4 |
+| T5-base | 61.2 | 78.1 | 80.5 | 66.2 | 48.0 | 50.3 | 62.0 | 49.4 | 66.0 | 47.1 | 61.4 |
+
+*MSGS*
+| Model | CR (Control) | LC (Control) | MV (Control) | RP (Control) | SC (Control) | CR_LC | CR_RTP | MV_LC | MV_RTP | SC_LC | SC_RP |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| OPT-125m | 86.4 | 86.1 | 99.8 | 100.0 | 94.3 | 66.5 | 67.0 | 66.5 | 67.6 | 80.2 | 67.5 |
+| RoBERTa-base | 84.1 | 100.0 | 99.4 | 93.5 | 96.4 | 67.7 | 68.6 | 66.7 | 68.6 | 84.2 | 65.7 | 
+| T5-base | 78.4 | 100.0 | 72.7 | 95.5 | 94.4 | 66.7 | 69.7 | 66.6 | 66.9 | 73.6 | 67.8 |
+
+*Age-of-acquisition Prediction*
+(Mean absolute deviation in months across LOO cross-validation folds)
+| Model | Overall (591 words) | Nouns (322) | Predicates (167) | Function words (102) |
+| --- | --- | --- | --- | --- |
+| OPT-125m | 2.03 | 1.98 | 1.81 | 2.57 |
+| RoBERTa-base | 2.06 | 1.99 | 1.85 | 2.65 |
+| T5-base | 2.04 | 1.97 | 1.82 | 2.64 |
+
+-------------
+
+**Strict Track**
+
+*BLiMP*
+| Model | Anaphor Agr. | Agr. Structure | Binding | Control/Raising | D-N Agr. | Ellipsis | Filler-Gap | Irregular Forms | Island Effects | NPI Licensing | Quantifiers | S-V Agr. |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| OPT-125m | 94.9 | 73.8 | 73.8 | 72.2 | 93.1 | 80.5 | 73.6 | 80.8 | 57.8 | 51.6 | 74.5 | 77.3 |
+| RoBERTa-base | 89.5 | 71.3 | 71 | 67.1 | 93.1 | 83.8 | 68.0 | 89.6 | 54.5 | 66.3 | 70.3 | 76.2 |
+| T5-base | 66.7 | 61.2 | 59.4 | 59.8 | 53.8 | 49.1 | 70.0 | 75.5 | 43.6 | 45.6 | 34.2 | 53.2 |
+
+*BLiMP Supplement*
+| Model | Hypernym | QA Congruence (easy) | QA Congruence (tricky) | Subj.-Aux. Inversion | Turn Taking |
+| --- | --- | --- | --- | --- | --- |
+| OPT-125m | 46.3 | 76.5 | 47.9 | 85.3 | 82.9 |
+| RoBERTa-base | 50.8 | 34.4 | 34.5 | 45.6 | 46.8 |
+| T5-base | 51.1 | 45.3 | 25.5 | 69.2 | 48.9 |
+
+*(Super)GLUE*
+| Model | CoLA | SST-2 | MRPC (F1) | QQP (F1) | MNLI | MNLI-mm | QNLI | RTE | BoolQ | MultiRC | WSC |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| *Majority label* | *69.5* | *50.2* | *82* | *53.1* | *35.7* | *35.7* | *35.4* | *53.1* | *50.5* | *59.9* | *53.2* | *61.4* |
+| OPT-125m | 73.7 | 86.6 | 82.1 | 77.8 | 70.1 | 71.9 | 80.1 | 67.7 | 66.0 | 61.1 | 59.0 |
+| RoBERTa-base | 75.9 | 88.6 | 80.5 | 78.5 | 68.7 | 78.0 | 82.3 | 51.5 | 59.9 | 61.3 | 61.4 |
+| T5-base | 76.3 | 88.0 | 85.9 | 79.7 | 71.5 | 74.0 | 83.1 | 60.6 | 69.0 | 62.4 | 60.2 |
+
+*MSGS*
+| Model | CR (Control) | LC (Control) | MV (Control) | RP (Control) | SC (Control) | CR_LC | CR_RTP | MV_LC | MV_RTP | SC_LC | SC_RP |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| OPT-125m | 97.2 | 82.6 | 100.0 | 99.8 | 88.1 | 75.3 | 67.1 | 66.3 | 66.8 | 84.8 | 62.0 | 
+| RoBERTa-base | 93.0 | 100.0 | 100.0 | 100.0 | 89.0 | 68.3 | 66.8 | 66.6 | 80.2 | 67.4 | 67.4 | 
+| T5-base | 95.1 | 100.0 | 100.0 | 99.8 | 88.7 | 76.7 | 69.4 | 67.0 | 67.7 | 72.7 | 68.0 |
+
+-----------------------
+
+These are naïve baselines that are meant to provide a starting point for investigation. We look forward to seeing how you will improve upon these!
+
+## Citation
+If you use the datasets or code from this repository, please cite the BabyLM Call for Papers:
+
+```
+@article{warstadt2023papers,
+      title     = {Call for Papers -- The BabyLM Challenge: Sample-efficient pretraining on a developmentally plausible corpus},
+      author    = {Warstadt, Alex and
+                   Choshen, Leshem and
+                   Mueller, Aaron and
+                   Williams, Adina and
+                   Wilcox, Ethan and
+                   Zhuang, Chengxu},
+      year      = {2023},
+      journal   = {Computing Research Repository},
+      volume    = {arXiv:2301.11796}
+}
+```
+
+Please also cite the lm-eval-harness paper:
+```
+@software{eval-harness,
+  author       = {Gao, Leo and
+                  Tow, Jonathan and
+                  Biderman, Stella and
+                  Black, Sid and
+                  DiPofi, Anthony and
+                  Foster, Charles and
+                  Golding, Laurence and
+                  Hsu, Jeffrey and
+                  McDonell, Kyle and
+                  Muennighoff, Niklas and
+                  Phang, Jason and
+                  Reynolds, Laria and
+                  Tang, Eric and
+                  Thite, Anish and
+                  Wang, Ben and
+                  Wang, Kevin and
+                  Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = sep,
+  year         = 2021,
+  publisher    = {Zenodo},
+  version      = {v0.0.1},
+  doi          = {10.5281/zenodo.5371628},
+  url          = {https://doi.org/10.5281/zenodo.5371628}
+}
+```
+
+Please cite the following if you choose to include the Age-of-acquisition prediction evaluation:
+```
+@manuscript{portelance2023predicting,
+    author = {Portelance, Eva and Duan, Yuguang and Frank, Michael C. and Lupyan, Gary},
+    title = {Predicting age of acquisition for children’s early vocabulary in five languages using language model surprisal},
+    year = {2023},
+    url = {https://github.com/evaportelance/multilingual-aoa-prediction}
+    }

evaluation-pipeline/aoa_data/word_list.csv

@@ -0,0 +1,592 @@
+language,uni_lemma,lexical_category,category,definition,word_clean,aoa,concreteness,frequency
+English (American),a,function_words,quantifiers,a,a,26.75657271813926,1.46,301773
+English (American),a,function_words,quantifiers,an,an,26.75657271813926,1.46,13508
+English (American),a lot,function_words,quantifiers,a lot,a lot,29.354943884668234,2.37,4807
+English (American),about,function_words,locations,about,about,33.56541793707104,1.77,27478
+English (American),above,function_words,locations,above,above,32.94888134846138,3.33,332
+English (American),airplane,nouns,vehicles,airplane,airplane,20.14529714809586,4.96,1192
+English (American),all,function_words,quantifiers,all,all,26.31156607337898,2.27,49585
+English (American),all gone,predicates,descriptive_words,all gone,all gone,19.106125409049845,2.04,779
+English (American),alligator,nouns,animals,alligator,alligator,25.74050182260553,4.96,404
+English (American),am,predicates,helping_predicates,am,am,29.39866638780256,1.96,4070
+English (American),and,function_words,connecting_words,and,and,27.669136574125037,1.52,217485
+English (American),animal,nouns,animals,animal,animal,24.531890826243654,4.61,1523
+English (American),ankle,nouns,body_parts,ankle,ankle,29.54373992549591,4.81,75
+English (American),another,function_words,quantifiers,another,another,28.49498165323644,2.69,14222
+English (American),ant,nouns,animals,ant,ant,24.632024173206105,4.86,182
+English (American),any,function_words,quantifiers,any,any,30.168894359149128,1.72,12098
+English (American),apple,nouns,food_drink,apple,apple,18.316415297779947,5,3736
+English (American),applesauce,nouns,food_drink,applesauce,applesauce,25.18708208845118,4.96,127
+English (American),are,function_words,helping_predicates,are,are,30.400848330016352,1.96,106583
+English (American),arm,nouns,body_parts,arm,arm,21.679117916792883,4.96,1528
+English (American),around,function_words,locations,around,around,29.73724069586142,1.96,6835
+English (American),asleep,predicates,descriptive_words,asleep,asleep,24.95132838125333,3.71,1365
+English (American),at,function_words,locations,at,at,28.44395214386389,2.07,52573
+English (American),awake,predicates,descriptive_words,awake,awake,26.406667306484906,3.32,362
+English (American),away,function_words,locations,away,away,27.268694284819865,2.23,11014
+English (American),back (location),function_words,locations,back,back,26.041013448918488,4.33,26533
+English (American),bad,predicates,descriptive_words,bad,bad,24.376033345405514,1.68,2576
+English (American),ball,nouns,toys,ball,ball,14.051831559140178,5,8382
+English (American),balloon,nouns,toys,balloon,balloon,17.252991342080126,4.92,1810
+English (American),banana,nouns,food_drink,banana,banana,16.787250784202772,5,1615
+English (American),basement,nouns,furniture_rooms,basement,basement,31.403406659378565,4.89,121
+English (American),basket,nouns,household,basket,basket,25.32137110090939,5,1734
+English (American),bat,nouns,toys,bat,bat,25.94740000625428,5,384
+English (American),bathroom,nouns,furniture_rooms,bathroom,bathroom,23.22834292081228,4.52,1059
+English (American),bathtub,nouns,furniture_rooms,bathtub,bathtub,21.86077443068652,4.92,410
+English (American),be,function_words,helping_predicates,be,be,30.344096265123305,1.85,41097
+English (American),beach,nouns,outside,beach,beach,26.078273320304067,4.79,991
+English (American),beads,nouns,clothing,beads,beads,29.819752587655973,4.9,546
+English (American),beans,nouns,food_drink,beans,beans,24.532090787380607,5,1249
+English (American),bear,nouns,animals,bear,bear,19.290426322598368,4.88,5220
+English (American),because,function_words,connecting_words,because,because,30.412366644638137,1.22,31730
+English (American),bed,nouns,furniture_rooms,bed,bed,20.494046155718603,5,7584
+English (American),bedroom,nouns,furniture_rooms,bedroom,bedroom,25.645644681903754,4.9,1119
+English (American),bee,nouns,animals,bee,bee,21.306296871726428,4.88,1475
+English (American),behind,function_words,locations,behind,behind,29.581051780523048,3.48,2954
+English (American),belly button,nouns,body_parts,belly button,belly button,20.62619109370771,5,90
+English (American),belt,nouns,clothing,belt,belt,25.39428676791078,4.9,307
+English (American),bench,nouns,furniture_rooms,bench,bench,31.122752354523378,4.87,241
+English (American),beside,function_words,locations,beside,beside,33.68775817791739,2.59,304
+English (American),better,predicates,descriptive_words,better,better,27.197801378663197,1.91,9742
+English (American),bib,nouns,clothing,bib,bib,23.648183674597796,4.79,639
+English (American),bicycle,nouns,vehicles,bicycle,bicycle,21.77871249104521,4.89,660
+English (American),big,predicates,descriptive_words,big,big,22.25931777062911,3.66,25182
+English (American),bird,nouns,animals,bird,bird,17.224492119324236,5,2517
+English (American),bite,predicates,action_words,bite,bite,21.93843222383973,4.44,2541
+English (American),black,predicates,descriptive_words,black,black,26.65032453394826,3.76,2775
+English (American),blanket,nouns,household,blanket,blanket,20.364584241108936,5,957
+English (American),block,nouns,toys,block,block,21.47767824557252,4.48,1117
+English (American),blow,predicates,action_words,blow,blow,24.100630216525534,3.74,2110
+English (American),blue,predicates,descriptive_words,blue,blue,23.520465305593365,3.76,9232
+English (American),boat,nouns,vehicles,boat,boat,20.384388304955454,4.93,2544
+English (American),book,nouns,toys,book,book,16.378840412498462,4.9,15413
+English (American),boots,nouns,clothing,boots,boots,22.762953333636315,5,792
+English (American),bottle,nouns,household,bottle,bottle,18.109441114425163,4.91,3023
+English (American),bowl,nouns,household,bowl,bowl,22.67450309415727,4.87,1967
+English (American),box,nouns,household,box,box,22.36283488674821,4.9,8714
+English (American),bread,nouns,food_drink,bread,bread,21.714838666180317,4.92,2959
+English (American),break,predicates,action_words,break,break,24.498575833372556,3.71,2940
+English (American),bring,predicates,action_words,bring,bring,27.00190998469981,2.55,5164
+English (American),broken,predicates,descriptive_words,broken,broken,23.262899073559634,4.11,2733
+English (American),broom,nouns,household,broom,broom,23.51056906797502,4.89,282
+English (American),brown,predicates,descriptive_words,brown,brown,27.44722287259727,4.48,2294
+English (American),brush (object),nouns,household,brush,brush,21.812402640756495,4.54,2162
+English (American),bubbles,nouns,toys,bubbles,bubbles,18.447326266810368,4.6,1229
+English (American),bucket,nouns,household,bucket,bucket,26.11855610121108,4.96,896
+English (American),bug,nouns,animals,bug,bug,21.45346928441042,5,653
+English (American),build,predicates,action_words,build,build,27.561167006392587,3.71,2552
+English (American),bump,predicates,action_words,bump,bump,25.507631046994266,4.1,1008
+English (American),bunny,nouns,animals,bunny,bunny,20.465056141535673,4.97,2386
+English (American),bus,nouns,vehicles,bus,bus,21.261211792115443,4.9,3774
+English (American),but,function_words,connecting_words,but,but,32.30802214785319,2.04,34537
+English (American),butter,nouns,food_drink,butter,butter,24.566501758151283,4.9,1358
+English (American),butterfly,nouns,animals,butterfly,butterfly,23.47450828971274,4.93,882
+English (American),button,nouns,clothing,button,button,22.084011296770807,4.96,1602
+English (American),buy,predicates,action_words,buy,buy,27.61820820612325,3.35,3593
+English (American),by,function_words,locations,by,by,28.916261166336813,1.55,5847
+English (American),cake,nouns,food_drink,cake,cake,21.93745596797483,4.81,3506
+English (American),camera,nouns,household,camera,camera,25.73750398456283,5,1185
+English (American),can (auxiliary),function_words,helping_predicates,can (auxiliary),can,27.82698008632811,4.55,175734
+English (American),can (object),nouns,household,can (object),can,26.84891293682625,4.55,175734
+English (American),candy,nouns,food_drink,candy,candy,22.10696542716163,4.83,704
+English (American),car,nouns,vehicles,car,car,17.53434303192908,4.89,13476
+English (American),careful,predicates,descriptive_words,careful,careful,26.43412690050291,1.86,6107
+English (American),carrots,nouns,food_drink,carrots,carrots,24.037122543158606,5,802
+English (American),carry,predicates,action_words,carry,carry,25.681963506798628,4.04,1148
+English (American),cat,nouns,animals,cat,cat,17.67814164845057,4.86,5835
+English (American),catch,predicates,action_words,catch,catch,25.556075994667953,4.11,2813
+English (American),cereal,nouns,food_drink,cereal,cereal,22.086966073355658,4.83,1034
+English (American),chair,nouns,furniture_rooms,chair,chair,20.747062695367397,4.58,6382
+English (American),chalk,nouns,toys,chalk,chalk,27.572948668482248,4.9,246
+English (American),chase,predicates,action_words,chase,chase,28.40000722125352,3.48,251
+English (American),cheek,nouns,body_parts,cheek,cheek,23.639051249460564,4.83,159
+English (American),cheerios,nouns,food_drink,cheerios,cheerios,23.30250911956345,4.35,59
+English (American),cheese,nouns,food_drink,cheese,cheese,18.35624365220089,4.7,4215
+English (American),chicken (animal),nouns,animals,chicken (animal),chicken,22.94116221978306,4.8,5636
+English (American),chicken (food),nouns,food_drink,chicken (food),chicken,22.755311666095626,4.8,5636
+English (American),chin,nouns,body_parts,chin,chin,23.932790792488237,4.89,444
+English (American),chips,nouns,food_drink,potato chip,potato chip,23.914163430448284,4.9,23
+English (American),chocolate,nouns,food_drink,chocolate,chocolate,25.207246056049254,4.72,2469
+English (American),church,nouns,outside,church*,church,26.377623217552873,4.9,547
+English (American),clap,predicates,action_words,clap,clap,23.57129610232303,4.16,1106
+English (American),clean (action),predicates,action_words,clean (action),clean,24.56600009616096,3.07,7338
+English (American),clean (description),predicates,descriptive_words,clean (description),clean,24.16485158037349,3.07,7338
+English (American),climb,predicates,action_words,climb,climb,25.941366782032414,4.11,1076
+English (American),clock,nouns,household,clock,clock,22.729118690605997,5,1709
+English (American),close,predicates,action_words,close,close,24.612899480281406,3.2,3138
+English (American),closet,nouns,furniture_rooms,closet,closet,26.33273273898802,4.83,289
+English (American),cloud,nouns,outside,cloud,cloud,25.68327028006053,4.54,273
+English (American),coat,nouns,clothing,coat,coat,22.77758807471102,4.97,1191
+English (American),coffee,nouns,food_drink,coffee,coffee,24.733868049413292,4.81,2249
+English (American),coke,nouns,food_drink,coke,coke,28.250686610987113,4.83,184
+English (American),cold,predicates,descriptive_words,cold,cold,20.986347269894612,3.85,3891
+English (American),comb (object),nouns,household,comb,comb,24.521530115219505,5,639
+English (American),cook,predicates,action_words,cook,cook,25.20950610885458,4.32,1634
+English (American),cookie,nouns,food_drink,cookie,cookie,18.05879104299356,5,2651
+English (American),corn,nouns,food_drink,corn,corn,23.75808620462897,4.96,769
+English (American),couch,nouns,furniture_rooms,couch,couch,24.81633921615295,4.71,505
+English (American),could,function_words,helping_predicates,could,could,33.0915181485895,1.34,12880
+English (American),cover (action),predicates,action_words,cover,cover,28.093896291699608,4.23,902
+English (American),cow,nouns,animals,cow,cow,20.23330616534318,4.96,3496
+English (American),cracker,nouns,food_drink,cracker,cracker,19.128739975548182,4.96,471
+English (American),crayon,nouns,toys,crayon,crayon,22.958268446308498,4.87,1010
+English (American),crib,nouns,furniture_rooms,crib,crib,25.54633696377014,4.86,306
+English (American),cry,predicates,action_words,cry,cry,22.833033348617004,4,1408
+English (American),cup,nouns,household,cup,cup,19.458761353171624,5,5361
+English (American),cut,predicates,action_words,cut,cut,26.834383653830635,4.55,3650
+English (American),cute,predicates,descriptive_words,cute,cute,26.878579973455473,2.76,1371
+English (American),dance,predicates,action_words,dance,dance,23.594173710650814,4.32,1107
+English (American),dark,predicates,descriptive_words,dark,dark,26.374254558849202,4.29,1494
+English (American),deer,nouns,animals,deer,deer,26.28895723853603,4.86,374
+English (American),diaper,nouns,clothing,diaper,diaper,19.45888352722782,4.82,1072
+English (American),did,function_words,helping_predicates,did/did ya,did,28.236905169713484,2.45,57888
+English (American),dirty,predicates,descriptive_words,dirty,dirty,22.183512254438906,4.23,2706
+English (American),dish,nouns,household,dish,dish,26.369638755777036,4.9,312
+English (American),do,function_words,helping_predicates,do,do,24.418127084458625,2.46,157341
+English (American),does,function_words,helping_predicates,does,does,31.321420112088582,2.24,30079
+English (American),dog,nouns,animals,dog,dog,14.594052719227676,4.85,5509
+English (American),doll,nouns,toys,doll,doll,21.41564709887175,5,1830
+English (American),don't,function_words,helping_predicates,don't,dont,25.29034229368797,2.46,89665
+English (American),donkey,nouns,animals,donkey,donkey,27.89414096656381,5,575
+English (American),donut,nouns,food_drink,donut,donut,26.088004339085252,4.93,98
+English (American),door,nouns,furniture_rooms,door,door,20.2792407612697,4.81,6193
+English (American),down,function_words,locations,down,down,19.778758085736765,3.52,31240
+English (American),draw,predicates,action_words,draw,draw,26.127344086880825,3.97,4917
+English (American),drawer,nouns,furniture_rooms,drawer,drawer,27.477050929051146,4.67,511
+English (American),dress (object),nouns,clothing,dress (object),dress,25.175475858275057,4.93,2087
+English (American),drink (action),predicates,action_words,drink (action),drink,21.405832133186447,3.77,12546
+English (American),drink (beverage),nouns,food_drink,drink (beverage),drink,21.12191463243953,4.76,12546
+English (American),drive,predicates,action_words,drive,drive,24.75429950810191,3.86,1883
+English (American),drop,predicates,action_words,drop,drop,26.14791694487336,4.21,1105
+English (American),dry (action),predicates,action_words,dry (action),dry,26.69818180786187,3.77,3410
+English (American),dry (description),predicates,descriptive_words,dry (description),dry,26.486133398623007,3.77,3410
+English (American),dryer,nouns,furniture_rooms,dryer,dryer,27.940244672930877,4.79,93
+English (American),duck,nouns,animals,duck,duck,17.185401989712528,4.86,2872
+English (American),dump,predicates,action_words,dump,dump,29.574191643049147,3.93,626
+English (American),each,function_words,quantifiers,each,each,34.433173955126186,2.03,1275
+English (American),ear,nouns,body_parts,ear,ear,18.42450761164981,5,1260
+English (American),eat,predicates,action_words,eat,eat,19.996890077067878,4.44,18091
+English (American),egg,nouns,food_drink,egg,egg,22.122988028871536,4.97,2550
+English (American),elephant,nouns,animals,elephant,elephant,23.021554163445344,5,2858
+English (American),empty,predicates,descriptive_words,empty,empty,26.56355703829002,3.43,1237
+English (American),every,function_words,quantifiers,every,every,32.62625782784631,2.28,1932
+English (American),eye,nouns,body_parts,eye,eye,17.19831986641664,4.9,1696
+English (American),face,nouns,body_parts,face,face,23.386576805200583,4.87,4356
+English (American),fall,predicates,action_words,fall,fall,23.05658646359307,4.04,4547
+English (American),fast,predicates,descriptive_words,fast,fast,26.397899335243192,3.32,1814
+English (American),feed,predicates,action_words,feed,feed,27.04318304347576,4.17,1696
+English (American),find,predicates,action_words,find,find,25.911291431365097,2.63,11096
+English (American),fine,predicates,descriptive_words,fine,fine,30.348259169087953,2.61,1791
+English (American),finger,nouns,body_parts,finger,finger,21.846032665491855,5,2157
+English (American),finish,predicates,action_words,finish,finish,28.14761497515676,2.89,2774
+English (American),firetruck,nouns,vehicles,firetruck,firetruck,24.716233162853985,5,149
+English (American),first,predicates,descriptive_words,first,first,28.530149212482925,2.76,8389
+English (American),fish (animal),nouns,animals,fish (animal),fish,18.978116085220503,5,9298
+English (American),fish (food),nouns,food_drink,fish (food),fish,22.085905665472275,5,9298
+English (American),fit,predicates,action_words,fit,fit,29.08064200730688,2.7,3997
+English (American),fix,predicates,action_words,fix,fix,25.32658824937031,2.93,2929
+English (American),flag,nouns,outside,flag,flag,26.39411115381629,4.79,236
+English (American),flower,nouns,outside,flower,flower,20.519981351232357,5,1361
+English (American),food,nouns,food_drink,food,food,23.490400956729655,4.8,6233
+English (American),foot,nouns,body_parts,foot,foot,20.351627730476352,4.9,3505
+English (American),for,function_words,locations,for,for,29.21964285739064,1.63,60043
+English (American),fork,nouns,household,fork,fork,21.912707160579416,4.9,782
+English (American),french fries,nouns,food_drink,french fries,french fries,22.485372598777225,4.87,111
+English (American),frog,nouns,animals,frog,frog,22.53087514553091,5,1734
+English (American),full,predicates,descriptive_words,full,full,27.307650021326054,3.59,1585
+English (American),game,nouns,toys,game,game,26.558324042448465,4.5,2813
+English (American),garage,nouns,furniture_rooms,garage,garage,26.95193574573295,4.96,1135
+English (American),garbage,nouns,household,garbage,garbage,25.263290629707857,4.69,493
+English (American),garden,nouns,outside,garden,garden,28.814903721740233,4.73,1711
+English (American),gentle,predicates,descriptive_words,gentle,gentle,27.53063486592105,2.53,955
+English (American),get,predicates,action_words,get,get,24.203818250784888,2.38,55665
+English (American),giraffe,nouns,animals,giraffe,giraffe,24.848340134105648,4.73,1156
+English (American),give,predicates,action_words,give,give,26.131793055183277,2.83,13726
+English (American),glass,nouns,household,glass,glass,25.377947756405593,4.82,1069
+English (American),glasses,nouns,household,glasses,glasses,23.382539516196495,4.9,985
+English (American),gloves,nouns,clothing,gloves,gloves,27.24887506361188,4.97,339
+English (American),glue,nouns,toys,glue,glue,29.779166021342032,4.65,672
+English (American),go,predicates,action_words,go,go,19.73419022328145,3.15,83559
+English (American),good,predicates,descriptive_words,good,good,23.555059836026327,1.64,42780
+English (American),goose,nouns,animals,goose,goose,26.868415336421975,4.81,522
+English (American),grapes,nouns,food_drink,grapes,grapes,21.87000053388467,5,1148
+English (American),grass,nouns,outside,grass,grass,23.421522474544503,4.93,1407
+English (American),green,predicates,descriptive_words,green,green,24.864917218093773,4.07,8568
+English (American),green beans,nouns,food_drink,green beans,green beans,28.18246325512689,4.9,224
+English (American),gum,nouns,food_drink,gum,gum,25.6705889024626,4.89,374
+English (American),hair,nouns,body_parts,hair,hair,19.550956015647447,4.97,6378
+English (American),hamburger,nouns,food_drink,hamburger,hamburger,24.060486137223638,5,565
+English (American),hammer,nouns,household,hammer,hammer,25.908134489269617,4.77,860
+English (American),hand,nouns,body_parts,hand,hand,20.888392084906826,4.72,6426
+English (American),happy,predicates,descriptive_words,happy,happy,24.111067699312272,2.56,3476
+English (American),hard,predicates,descriptive_words,hard,hard,28.34690319460537,3.76,4563
+English (American),hat,nouns,clothing,hat,hat,18.800006578327608,4.88,5643
+English (American),hate,predicates,action_words,hate,hate,32.742713404389164,1.97,285
+English (American),have,predicates,action_words,have,have,26.383343346266226,2.18,80189
+English (American),he,function_words,pronouns,he,he,28.39328352610605,3.93,85599
+English (American),head,nouns,body_parts,head,head,21.52867556120851,4.75,7286
+English (American),hear,predicates,action_words,hear,hear,27.103048512621903,3.66,6278
+English (American),heavy,predicates,descriptive_words,heavy,heavy,25.14874997804902,3.37,1182
+English (American),helicopter,nouns,vehicles,helicopter,helicopter,24.90634156578962,4.62,729
+English (American),help,predicates,action_words,help,help,22.819438880083318,2.56,8981
+English (American),hen,nouns,animals,hen,hen,29.623281978610162,4.9,378
+English (American),her,function_words,pronouns,her,her,30.0706499832329,3,32518
+English (American),here,function_words,locations,here,here,25.0324996525398,3.13,74502
+English (American),hers,function_words,pronouns,hers,hers,31.577356326543732,2.61,401
+English (American),hide,predicates,action_words,hide,hide,25.6831882822722,3.21,1461
+English (American),high,predicates,descriptive_words,high,high,27.021723647237135,3.46,1690
+English (American),high chair,nouns,furniture_rooms,high chair,high chair,25.143576833379374,4.83,177
+English (American),him,function_words,pronouns,him,him,30.171545513305684,3.54,24073
+English (American),his,function_words,pronouns,his,his,30.86529830099037,3.14,25985
+English (American),hit,predicates,action_words,hit,hit,24.176504122997176,4.11,2339
+English (American),hold,predicates,action_words,hold,hold,25.28951191411586,3.68,6702
+English (American),home,nouns,outside,home,home,21.55292850686028,4.11,9648
+English (American),horse,nouns,animals,horse,horse,20.598917207238273,5,4162
+English (American),hose,nouns,outside,hose,hose,26.925240566139266,4.87,264
+English (American),hot,predicates,descriptive_words,hot,hot,17.886174346828543,4.31,4425
+English (American),house,nouns,outside,house,house,22.949798098688984,5,12111
+English (American),how,function_words,question_words,how,how,30.185466664457547,1.35,31682
+English (American),hug,predicates,action_words,hug,hug,22.04054246795061,4.14,1172
+English (American),hungry,predicates,descriptive_words,hungry,hungry,24.529656637397714,2.9,2717
+English (American),hurry,predicates,action_words,hurry,hurry,27.48425052166916,2.63,865
+English (American),hurt (description),predicates,descriptive_words,hurt,hurt,24.88161558121455,3.61,4613
+English (American),i,function_words,pronouns,I,i,23.977441347865177,3.93,786
+English (American),ice,nouns,food_drink,ice,ice,22.358270283361815,4.89,2197
+English (American),ice cream,nouns,food_drink,ice cream,ice cream,21.580952159819738,5,1557
+English (American),if,function_words,connecting_words,if,if,34.068154850176526,1.19,31526
+English (American),in,function_words,locations,in,in,24.07962150691501,3,276036
+English (American),in,function_words,locations,inside/in,inside,24.07962150691501,3,7888
+English (American),inside,function_words,locations,inside,inside,28.910801367846076,3.67,7888
+English (American),into,function_words,locations,into,into,32.328072400666066,2.3,7862
+English (American),is,function_words,helping_predicates,is,is,28.508756906338796,1.59,183357
+English (American),it,function_words,pronouns,it,it,25.985519116000557,2.81,297679
+English (American),jacket,nouns,clothing,jacket,jacket,23.764267757552943,4.86,655
+English (American),jar,nouns,household,jar,jar,30.036814769417507,5,389
+English (American),jeans,nouns,clothing,jeans,jeans,27.85718213630826,5,160
+English (American),jelly,nouns,food_drink,jelly,jelly,26.459272534245788,4.93,851
+English (American),juice,nouns,food_drink,juice,juice,17.663424879988565,4.89,6380
+English (American),jump,predicates,action_words,jump,jump,23.05187845463102,4.52,2396
+English (American),keys,nouns,household,keys,keys,19.821178140120676,4.89,1273
+English (American),kick,predicates,action_words,kick,kick,24.607244736468125,4.33,925
+English (American),kiss,predicates,action_words,kiss,kiss,20.97729762630762,4.48,3370
+English (American),kitchen,nouns,furniture_rooms,kitchen,kitchen,23.721678813935878,4.97,2439
+English (American),kitty,nouns,animals,kitty,kitty,17.76353044822186,4.97,2807
+English (American),knee,nouns,body_parts,knee,knee,22.83341234944545,5,1252
+English (American),knife,nouns,household,knife,knife,25.180125523737622,4.9,749
+English (American),knock,predicates,action_words,knock,knock,25.65520970263338,4.24,1500
+English (American),ladder,nouns,outside,ladder,ladder,27.453139889383696,5,836
+English (American),lamb,nouns,animals,lamb,lamb,26.77611745254101,4.97,762
+English (American),lamp,nouns,household,lamp,lamp,27.707727325952707,4.97,286
+English (American),last,predicates,descriptive_words,last,last,31.94810812999018,3.04,6041
+English (American),lawn mower,nouns,outside,lawn mower,lawn mower,26.25206641047887,4.93,37
+English (American),leg,nouns,body_parts,leg,leg,22.786751225180204,4.83,1696
+English (American),lick,predicates,action_words,lick,lick,27.885918786410006,4.52,443
+English (American),light (object),nouns,household,light,light,20.167269067810214,4.21,2752
+English (American),like (action),predicates,action_words,like,like,25.712763627585442,1.89,80659
+English (American),lion,nouns,animals,lion,lion,23.292560818293822,4.96,1972
+English (American),listen,predicates,action_words,listen,listen,28.120047324042673,3.47,3194
+English (American),little (description),predicates,descriptive_words,little (description),little,25.166304259548355,3.67,39012
+English (American),living room,nouns,furniture_rooms,living room,living room,27.67343106925516,4.7,431
+English (American),lollipop,nouns,food_drink,lollipop,lollipop,27.20359032793206,4.96,276
+English (American),long,predicates,descriptive_words,long,long,30.43920077538411,3.18,5656
+English (American),look,predicates,action_words,look,look,24.07200126089267,2.96,68469
+English (American),loud,predicates,descriptive_words,loud,loud,27.096587104346657,3.73,1100
+English (American),love,predicates,action_words,love,love,22.97930051104563,2.07,5139
+English (American),mad,predicates,descriptive_words,mad,mad,28.322211878333118,2.76,839
+English (American),make,predicates,action_words,make,make,27.26683722139889,2.67,22678
+English (American),me,function_words,pronouns,me,me,21.782136908963487,4.33,58317
+English (American),meat,nouns,food_drink,meat,meat,25.21618096142793,4.9,986
+English (American),medicine,nouns,household,medicine,medicine,24.410637544584084,4.79,915
+English (American),melon,nouns,food_drink,melon,melon,26.929946230882774,4.78,141
+English (American),milk,nouns,food_drink,milk,milk,18.304538681163276,4.92,5537
+English (American),mine,function_words,pronouns,mine,mine,20.0092783421445,3.56,2670
+English (American),mittens,nouns,clothing,mittens,mittens,27.13676422394422,4.89,267
+English (American),money,nouns,household,money,money,22.84702188801892,4.54,3251
+English (American),monkey,nouns,animals,monkey,monkey,21.75146876358306,4.9,2316
+English (American),moon,nouns,outside,moon,moon,21.064582625649372,4.9,1461
+English (American),moose,nouns,animals,moose,moose,30.726348818983883,4.97,324
+English (American),mop,nouns,household,mop,mop,28.697523528698856,4.97,158
+English (American),more,function_words,quantifiers,more,more,19.934125196097916,2.37,23403
+English (American),motorcycle,nouns,vehicles,motorcycle,motorcycle,25.424516980060698,4.97,366
+English (American),mouse,nouns,animals,mouse,mouse,23.186364426729803,4.83,1784
+English (American),mouth,nouns,body_parts,mouth,mouth,20.385274871125226,4.74,6017
+English (American),much,function_words,quantifiers,much,much,31.135471437484984,1.69,7404
+English (American),muffin,nouns,food_drink,muffin,muffin,26.053662526327358,4.78,219
+English (American),my,function_words,pronouns,my,my,24.67561156823274,2.42,30474
+English (American),myself,function_words,pronouns,myself,myself,30.826981734088697,2.97,674
+English (American),nail (object),nouns,household,nail,nail,27.745919885712205,4.93,268
+English (American),napkin,nouns,household,napkin,napkin,24.71066092570806,4.93,393
+English (American),naughty,predicates,descriptive_words,naughty,naughty,30.45927863451864,2.04,2904
+English (American),necklace,nouns,clothing,necklace,necklace,25.67583784602768,4.96,341
+English (American),need,function_words,helping_predicates,need/need to,need,28.517326473959855,1.69,11101
+English (American),new,predicates,descriptive_words,new,new,28.242927764036946,2.81,5522
+English (American),nice,predicates,descriptive_words,nice,nice,24.731878757128417,2.18,20152
+English (American),noisy,predicates,descriptive_words,noisy,noisy,28.238292864810433,3.56,903
+English (American),none,function_words,quantifiers,none,none,30.075465085938674,2.59,494
+English (American),nose,nouns,body_parts,nose,nose,17.50000892567104,4.89,5607
+English (American),not,function_words,quantifiers,not,not,28.331503696333467,2.08,62788
+English (American),of,function_words,locations,of,of,32.135421130600605,1.67,72422
+English (American),off,function_words,locations,off,off,21.76331515371575,2.79,21580
+English (American),old,predicates,descriptive_words,old,old,29.745205894423886,2.72,3975
+English (American),on,function_words,locations,on,on,22.07453703332649,3.25,130164
+English (American),open (action),predicates,action_words,open,open,21.70959024951605,3.21,8246
+English (American),orange (description),predicates,descriptive_words,orange (description),orange,24.48307542955755,4.66,10036
+English (American),orange (food),nouns,food_drink,orange (food),orange,22.862362842722305,4.66,10036
+English (American),other,function_words,quantifiers,other,other,29.008003039766972,2.04,15644
+English (American),our,function_words,pronouns,our,our,31.879580552349164,1.97,6116
+English (American),out,function_words,locations,out,out,21.787538623492427,2.73,34288
+English (American),outside,nouns,outside,outside,outside,20.10026304783487,4.25,5164
+English (American),oven,nouns,furniture_rooms,oven,oven,26.959440376130473,4.97,718
+English (American),over,function_words,locations,over,over,27.81126335036735,2.46,23641
+English (American),owie,nouns,body_parts,owie/boo boo,owie,19.225364028172503,3.61,101
+English (American),owl,nouns,animals,owl,owl,24.378811443862723,4.93,737
+English (American),paint (action),predicates,action_words,paint,paint,26.7264001349599,4.79,1346
+English (American),pajamas,nouns,clothing,pajamas,pajamas,23.01441672978766,4.73,776
+English (American),pancake,nouns,food_drink,pancake,pancake,24.195947688413984,4.86,201
+English (American),pants,nouns,clothing,pants,pants,21.997653199154513,4.86,2028
+English (American),paper,nouns,household,paper,paper,22.40794784536237,4.93,4643
+English (American),park,nouns,outside,park,park,23.44395480659095,4.74,1855
+English (American),party,nouns,outside,party,party,26.48232416632208,3.89,2705
+English (American),pasta,nouns,food_drink,noodles,noodles,22.2125626020018,4.86,209
+English (American),pasta,nouns,food_drink,spaghetti,spaghetti,22.2125626020018,4.86,1466
+English (American),peanut butter,nouns,food_drink,peanut butter,peanut butter,24.48646845168075,4.93,323
+English (American),peas,nouns,food_drink,peas,peas,24.557921410443495,4.9,2307
+English (American),pen,nouns,toys,pen,pen,23.51987496977609,4.92,1411
+English (American),pencil,nouns,toys,pencil,pencil,25.06258390684458,4.88,1040
+English (American),penguin,nouns,animals,penguin,penguin,27.293224141508716,5,658
+English (American),penis,nouns,body_parts,penis*,penis,26.231902171762034,5,35
+English (American),penny,nouns,household,penny,penny,26.32129034179081,4.83,575
+English (American),pick,predicates,action_words,pick,pick,28.350959500123274,3.82,4884
+English (American),pickle,nouns,food_drink,pickle,pickle,25.496142784956177,4.64,191
+English (American),picture,nouns,household,picture,picture,24.228451852721275,4.52,6309
+English (American),pig,nouns,animals,pig,pig,21.04970478014681,5,2423
+English (American),pillow,nouns,household,pillow,pillow,22.179110907856753,5,683
+English (American),pizza,nouns,food_drink,pizza,pizza,21.33010054090465,5,2650
+English (American),plant,nouns,household,plant,plant,26.161845259337724,4.76,519
+English (American),plate,nouns,household,plate,plate,24.38848139227323,4.77,2600
+English (American),play,predicates,action_words,play,play,22.50225294745721,3.24,23148
+English (American),play dough,nouns,toys,play dough,play dough,27.200019866322975,4.68,124
+English (American),play pen,nouns,furniture_rooms,play pen,play pen,30.93414000000259,4.76,13
+English (American),pony,nouns,animals,pony,pony,27.149124759679015,4.9,253
+English (American),pool,nouns,outside,pool,pool,23.288489299463958,4.77,860
+English (American),poor,predicates,descriptive_words,poor,poor,32.66767129734731,2.7,2640
+English (American),popcorn,nouns,food_drink,popcorn,popcorn,23.68702745961217,5,195
+English (American),popsicle,nouns,food_drink,popsicle,popsicle,25.14044583723922,4.93,120
+English (American),porch,nouns,furniture_rooms,porch,porch,29.70353523171374,4.92,84
+English (American),potato,nouns,food_drink,potato,potato,25.045299416217585,4.85,594
+English (American),potty,nouns,furniture_rooms,potty,potty,21.163868967261262,4.12,991
+English (American),pour,predicates,action_words,pour,pour,28.27835611871483,4.14,1410
+English (American),present,nouns,toys,present,present,24.99147354255208,3.39,1400
+English (American),pretend,predicates,action_words,pretend,pretend,31.275328228201392,2.11,3413
+English (American),pretty,predicates,descriptive_words,pretty,pretty,23.70523132395096,2.4,6516
+English (American),pretzel,nouns,food_drink,pretzel,pretzel,27.086772724245172,4.74,85
+English (American),pudding,nouns,food_drink,pudding,pudding,28.336874661648956,4.9,306
+English (American),pull,predicates,action_words,pull,pull,26.41351492122096,3.97,4797
+English (American),pumpkin,nouns,food_drink,pumpkin,pumpkin,25.596056826431344,4.9,815
+English (American),puppy,nouns,animals,puppy,puppy,20.642644342886626,4.78,1308
+English (American),purse,nouns,household,purse,purse,24.680534115242512,4.9,487
+English (American),push,predicates,action_words,push,push,24.884039995017684,4.21,5386
+English (American),put,predicates,action_words,put,put,27.180432184212417,2.5,68322
+English (American),puzzle,nouns,toys,puzzle,puzzle,24.603177970245078,4.75,2062
+English (American),quiet,predicates,descriptive_words,quiet,quiet,27.05139131191479,2.76,1380
+English (American),radio,nouns,household,radio,radio,26.613053034699238,4.74,355
+English (American),rain,nouns,outside,rain,rain,22.05272569057193,4.97,1240
+English (American),raisin,nouns,food_drink,raisin,raisin,23.922862069518597,4.97,169
+English (American),read,predicates,action_words,read,read,22.864582930368915,3.56,9185
+English (American),red,predicates,descriptive_words,red,red,24.476663524110176,4.24,9917
+English (American),refrigerator,nouns,furniture_rooms,refrigerator,refrigerator,25.673848500166034,5,562
+English (American),ride,predicates,action_words,ride,ride,24.3290998464415,3.75,2928
+English (American),rip,predicates,action_words,rip,rip,30.898483534092605,3.79,375
+English (American),rock (object),nouns,outside,rock,rock,22.092980833031184,4.91,875
+English (American),rocking chair,nouns,furniture_rooms,rocking chair,rocking chair,26.286720661876167,4.93,184
+English (American),roof,nouns,outside,roof,roof,29.380455204401983,4.79,941
+English (American),room,nouns,furniture_rooms,room,room,24.7800417765674,4.79,5211
+English (American),rooster,nouns,animals,rooster,rooster,27.785104097477404,4.75,422
+English (American),run,predicates,action_words,run,run,23.547368180082476,4.31,2505
+English (American),sad,predicates,descriptive_words,sad,sad,26.975730544989975,3.07,1405
+English (American),salt,nouns,food_drink,salt,salt,28.441896714165853,4.89,431
+English (American),same,function_words,quantifiers,same,same,30.94396519191967,2.64,3923
+English (American),sandbox,nouns,outside,sandbox,sandbox,26.956125375322088,4.86,172
+English (American),sandwich,nouns,food_drink,sandwich,sandwich,24.33836806143716,4.9,1683
+English (American),sauce,nouns,food_drink,sauce,sauce,27.988140408941376,4.75,487
+English (American),say,predicates,action_words,say,say,27.350714856102098,2.58,29942
+English (American),scared,predicates,descriptive_words,scared,scared,27.284769726224756,2.5,1001
+English (American),scarf,nouns,clothing,scarf,scarf,31.0680540145124,4.97,254
+English (American),school,nouns,outside,school,school,23.424986892471757,4.79,6710
+English (American),scissors,nouns,household,scissors,scissors,25.34069479638193,4.85,742
+English (American),see,predicates,action_words,see,see,22.628651743930725,3.21,71442
+English (American),shake,predicates,action_words,shake,shake,27.417577200447113,4.07,2018
+English (American),share,predicates,action_words,share,share,26.777735575736017,2.96,1061
+English (American),she,function_words,pronouns,she,she,29.293434421039155,3.36,53246
+English (American),sheep,nouns,animals,sheep,sheep,23.925866583400786,4.9,2362
+English (American),shirt,nouns,clothing,shirt,shirt,21.792127463554,4.94,2380
+English (American),shoe,nouns,clothing,shoe,shoe,16.401616618833103,4.97,2169
+English (American),shorts,nouns,clothing,shorts,shorts,24.909759184344438,4.82,425
+English (American),shoulder,nouns,body_parts,shoulder,shoulder,27.357714981859722,4.93,225
+English (American),shovel,nouns,outside,shovel,shovel,25.93264008363907,4.97,291
+English (American),show,predicates,action_words,show,show,27.85277401420022,3.97,8568
+English (American),shower,nouns,furniture_rooms,shower,shower,23.85617473782963,4.89,513
+English (American),sick,predicates,descriptive_words,sick,sick,27.025423107263375,2.97,1207
+English (American),sidewalk,nouns,outside,sidewalk,sidewalk,27.982232325403388,4.96,104
+English (American),sing,predicates,action_words,sing,sing,24.699537103250435,4.34,3851
+English (American),sink,nouns,furniture_rooms,sink,sink,25.65056395838028,4.74,560
+English (American),sit,predicates,action_words,sit,sit,21.750618675138146,4.8,14108
+English (American),skate,predicates,action_words,skate,skate,30.980023419231532,4.56,55
+English (American),sky,nouns,outside,sky,sky,24.179550783288896,4.45,1303
+English (American),sled,nouns,vehicles,sled,sled,29.4219079215242,5,90
+English (American),sleep,predicates,action_words,sleep,sleep,22.7674333859663,4.44,5010
+English (American),sleepy,predicates,descriptive_words,sleepy,sleepy,25.765911658239688,2.77,808
+English (American),slide (action),predicates,action_words,slide (action),slide,24.394104047595913,4.48,3556
+English (American),slide (object),nouns,outside,slide (object),slide,23.27441979107034,4.48,3556
+English (American),slipper,nouns,clothing,slipper,slipper,25.90935922531163,4.86,266
+English (American),slow,predicates,descriptive_words,slow,slow,29.462982166623352,3.28,515
+English (American),smile,predicates,action_words,smile,smile,27.022198912378336,4.5,498
+English (American),sneaker,nouns,clothing,sneaker,sneaker,28.458789287265894,4.69,22
+English (American),snow,nouns,outside,snow,snow,25.263272056571832,4.85,1575
+English (American),snowman,nouns,outside,snowman,snowman,27.67210714348651,4.64,577
+English (American),snowsuit,nouns,clothing,snowsuit,snowsuit,32.10541910633961,5,14
+English (American),so,function_words,connecting_words,so,so,31.12075549333711,1.42,44980
+English (American),soap,nouns,household,soap,soap,22.217921774180518,4.93,709
+English (American),sock,nouns,clothing,sock,sock,19.632046621854386,4.91,831
+English (American),soda,nouns,food_drink,soda/pop,soda,24.87205183714649,4.97,226
+English (American),sofa,nouns,furniture_rooms,sofa,sofa,29.394839148936164,4.9,450
+English (American),soft,predicates,descriptive_words,soft,soft,26.4496476384763,3.88,1358
+English (American),some,function_words,quantifiers,some,some,26.15437450894456,2.48,45706
+English (American),soup,nouns,food_drink,soup,soup,24.537539621938286,4.72,1288
+English (American),spill,predicates,action_words,spill,spill,26.85160156093012,4.07,741
+English (American),splash,predicates,action_words,splash,splash,25.60191719014938,4.19,709
+English (American),spoon,nouns,household,spoon,spoon,20.22002234723444,4.96,3524
+English (American),sprinkler,nouns,outside,sprinkler,sprinkler,28.75113806495684,4.89,52
+English (American),squirrel,nouns,animals,squirrel,squirrel,25.05065234245397,4.89,507
+English (American),stairs,nouns,furniture_rooms,stairs,stairs,24.47753335085588,5,1063
+English (American),stand,predicates,action_words,stand,stand,26.47937683978296,4.16,3558
+English (American),star,nouns,outside,star,star,22.957358462903812,4.69,1068
+English (American),stay,predicates,action_words,stay,stay,26.586593702088443,2.15,4403
+English (American),stick,nouns,outside,stick,stick,24.363668955777232,4.59,2642
+English (American),sticky,predicates,descriptive_words,sticky,sticky,26.569871206088944,3.59,1177
+English (American),stone,nouns,outside,stone,stone,29.654573348257873,4.72,306
+English (American),stop,predicates,action_words,stop,stop,22.813711423562257,3.68,5527
+English (American),store,nouns,outside,store,store,23.347629882628436,4.5,1651
+English (American),story,nouns,toys,story,story,24.92454125516168,3.3,5318
+English (American),stove,nouns,furniture_rooms,stove,stove,26.381537023983125,4.96,520
+English (American),strawberry,nouns,food_drink,strawberry,strawberry,24.304122215327734,5,840
+English (American),street,nouns,outside,street,street,25.6074840807896,4.75,987
+English (American),stroller,nouns,vehicles,stroller,stroller,24.603444785777125,4.96,227
+English (American),stuck,predicates,descriptive_words,stuck,stuck,25.790527319514812,3.55,3519
+English (American),sun,nouns,outside,sun,sun,23.386638497101277,4.83,1983
+English (American),sweater,nouns,clothing,sweater,sweater,24.620590384363602,4.78,310
+English (American),sweep,predicates,action_words,sweep,sweep,26.53419291818447,3.72,266
+English (American),swim,predicates,action_words,swim,swim,24.83257594574256,4.43,986
+English (American),swing (action),predicates,action_words,swing (action),swing,23.099517041407093,4.54,2158
+English (American),swing (object),nouns,outside,swing (object),swing,21.97332053501421,4.54,2158
+English (American),table,nouns,furniture_rooms,table,table,23.395064163035677,4.9,5798
+English (American),take,predicates,action_words,take,take,27.354093799677152,3.06,20398
+English (American),talk,predicates,action_words,talk,talk,26.172898281486297,4.07,4771
+English (American),tape,nouns,household,tape,tape,25.98869069431121,4.9,2967
+English (American),taste,predicates,action_words,taste,taste,28.306250651871636,4.07,1493
+English (American),tear,predicates,action_words,tear,tear,30.110713846716816,4.56,352
+English (American),teddybear,nouns,animals,teddybear,teddybear,22.457673785720814,4.87,513
+English (American),telephone,nouns,household,telephone,telephone,20.80553350305347,4.96,1280
+English (American),that,function_words,pronouns,that,that,24.03974924284265,1.54,219719
+English (American),the,function_words,quantifiers,the,the,27.71532064680694,1.43,431031
+English (American),their,function_words,pronouns,their,their,33.050453300967234,3.34,5083
+English (American),them,function_words,pronouns,them,them,31.425327261028436,3.04,41587
+English (American),then,function_words,connecting_words,then,then,32.94497320291506,1.44,43062
+English (American),there,function_words,locations,there,there,25.554374013411266,2.2,106697
+English (American),these,function_words,pronouns,these,these,28.91030410540727,2.03,20843
+English (American),they,function_words,pronouns,they,they,31.558011791688703,2.93,53290
+English (American),think,predicates,action_words,think,think,31.060977381002104,2.41,57446
+English (American),thirsty,predicates,descriptive_words,thirsty,thirsty,25.9814458268787,3.86,776
+English (American),this,function_words,pronouns,this,this,25.53668411833393,2.14,112409
+English (American),those,function_words,pronouns,those,those,30.71305685632802,2.32,19370
+English (American),throw,predicates,action_words,throw,throw,24.82027667799749,4.04,5585
+English (American),tickle,predicates,action_words,tickle,tickle,23.030865141786716,3.85,2094
+English (American),tiger,nouns,animals,tiger,tiger,23.97601642667219,5,2349
+English (American),tights,nouns,clothing,tights,tights,30.49526782372822,4.62,89
+English (American),tiny,predicates,descriptive_words,tiny,tiny,30.75118196119715,3.11,1454
+English (American),tired,predicates,descriptive_words,tired,tired,26.126063847560406,3,3628
+English (American),tissue,nouns,household,tissue/kleenex,tissue,24.881517554334437,4.93,915
+English (American),to,function_words,locations,to,to,27.829074756482076,1.55,183409
+English (American),toast,nouns,food_drink,toast,toast,22.957745051917446,4.93,2117
+English (American),toe,nouns,body_parts,toe,toe,21.063402869488534,4.93,596
+English (American),tongue,nouns,body_parts,tongue,tongue,23.578714641827492,4.93,776
+English (American),too,function_words,quantifiers,too,too,25.97451456969294,1.7,21317
+English (American),tooth,nouns,body_parts,tooth,tooth,21.390186244463926,4.89,351
+English (American),toothbrush,nouns,household,toothbrush,toothbrush,21.327972861784076,5,330
+English (American),touch,predicates,action_words,touch,touch,26.140740016873888,3.86,3096
+English (American),towel,nouns,household,towel,towel,23.309359030136616,4.86,942
+English (American),toy (object),nouns,toys,toy (object),toy,21.250379422750402,4.93,2764
+English (American),tractor,nouns,vehicles,tractor,tractor,26.293460903708173,5,1903
+English (American),train,nouns,vehicles,train,train,21.050773507323704,4.79,8225
+English (American),trash,nouns,household,trash,trash,24.623003619992428,4.7,297
+English (American),tray,nouns,household,tray,tray,31.297608014031766,4.74,570
+English (American),tree,nouns,outside,tree,tree,20.395692984405432,5,4310
+English (American),tricycle,nouns,vehicles,tricycle,tricycle,28.84180554601145,4.68,163
+English (American),truck,nouns,vehicles,truck,truck,19.04437063778738,4.84,4906
+English (American),try,function_words,helping_predicates,try/try to,try,28.751119300000134,2.22,8718
+English (American),tummy,nouns,body_parts,tummy,tummy,21.281768482103764,4.68,1772
+English (American),tuna,nouns,food_drink,tuna,tuna,28.649288768887633,4.89,208
+English (American),turkey,nouns,animals,turkey,turkey,26.30455622910081,4.89,611
+English (American),turtle,nouns,animals,turtle,turtle,23.014453310177558,5,792
+English (American),tv,nouns,furniture_rooms,TV,tv,21.36643906838051,5,1285
+English (American),under,function_words,locations,under,under,27.467319313886602,3.45,4364
+English (American),underpants,nouns,clothing,underpants,underpants,27.702339180277328,4.89,111
+English (American),up,function_words,locations,up,up,19.536229246282858,3.83,54966
+English (American),us,function_words,pronouns,us,us,32.68418082412553,3.59,7432
+English (American),vacuum,nouns,household,vacuum,vacuum,24.31715488103081,4.22,241
+English (American),vagina,nouns,body_parts,vagina*,vagina,30.74482822160873,4.82,8
+English (American),vanilla,nouns,food_drink,vanilla,vanilla,31.134282822644582,4.68,196
+English (American),vitamins,nouns,food_drink,vitamins,vitamins,26.849949095931386,4.5,85
+English (American),wait,predicates,action_words,wait,wait,26.620371092775404,2.68,7959
+English (American),wake,predicates,action_words,wake,wake,27.154377902728353,3.11,1167
+English (American),walk,predicates,action_words,walk,walk,22.817811861035416,4.07,3506
+English (American),walker,nouns,household,walker,walker,31.785543166543064,4.42,60
+English (American),wanna,function_words,helping_predicates,wanna/want to,wanna,25.18163215638973,1.93,45282
+English (American),was,function_words,helping_predicates,was,was,31.797860743565355,1.69,47639
+English (American),wash,predicates,action_words,wash,wash,23.987564663405482,4.35,3451
+English (American),washing machine,nouns,furniture_rooms,washing machine,washing machine,27.543044986998385,4.89,239
+English (American),watch (action),predicates,action_words,watch (action),watch,25.924613964341088,4.61,15632
+English (American),watch (object),nouns,household,watch (object),watch,24.00259582021144,4.61,15632
+English (American),water (beverage),nouns,food_drink,water (beverage),water,19.123463892055845,5,17388
+English (American),water (not beverage),nouns,outside,water (not beverage),water,19.45099861331479,5,17388
+English (American),we,function_words,pronouns,we,we,30.182434631151455,3.08,111064
+English (American),were,function_words,helping_predicates,were,were,33.709463033181024,1.48,30396
+English (American),wet (description),predicates,descriptive_words,wet,wet,22.298452425741512,4.46,2542
+English (American),what,function_words,question_words,what,what,23.658782166516907,2,205010
+English (American),when (question),function_words,question_words,when,when,31.814907045715227,1.6,27792
+English (American),where (question),function_words,question_words,where,where,25.7164381182961,1.66,38793
+English (American),which (question),function_words,question_words,which,which,33.63776900674318,1.54,10214
+English (American),white,predicates,descriptive_words,white,white,27.648764293432254,3.89,3496
+English (American),who,function_words,question_words,who,who,27.953549755744795,1.74,20283
+English (American),why,function_words,question_words,why,why,27.262555741780712,1.86,22267
+English (American),will,function_words,helping_predicates,will,will,30.909905381379183,2.64,14927
+English (American),wind,nouns,outside,wind,wind,26.19931053900361,3.93,1298
+English (American),window,nouns,furniture_rooms,window,window,24.48492597146611,4.86,2662
+English (American),windy,predicates,descriptive_words,windy,windy,27.431143183092633,3.86,342
+English (American),wipe,predicates,action_words,wipe,wipe,25.95723852087949,4,1809
+English (American),wish,predicates,action_words,wish,wish,32.96609018388594,1.77,669
+English (American),with,function_words,locations,with,with,28.56625914577001,2,64176
+English (American),wolf,nouns,animals,wolf,wolf,28.251389709841927,4.79,431
+English (American),work (action),predicates,action_words,work (action),work,24.762232083800356,3.48,12134
+English (American),work (place),nouns,outside,work (place),work,24.024006606906344,3.48,12134
+English (American),would,function_words,helping_predicates,would,would,34.17580948877974,1.12,22070
+English (American),write,predicates,action_words,write,write,26.86079876248854,4.22,2971
+English (American),yard,nouns,outside,backyard,backyard,26.10807276380005,4.82,105
+English (American),yellow,predicates,descriptive_words,yellow,yellow,24.5595931762692,4.3,6782
+English (American),yogurt,nouns,food_drink,yogurt,yogurt,23.91372922424347,4.9,589
+English (American),you,function_words,pronouns,you,you,24.120602892947364,4.11,589501
+English (American),your,function_words,pronouns,your,your,29.787082244817054,2.37,105998
+English (American),yourself,function_words,pronouns,yourself,yourself,34.28546272279592,4.39,3279
+English (American),yucky,predicates,descriptive_words,yucky,yucky,22.051017838612175,1.86,724
+English (American),zebra,nouns,animals,zebra,zebra,25.31795862018206,4.86,902
+English (American),zipper,nouns,clothing,zipper,zipper,24.84585073879806,4.83,421
+English (American),zoo,nouns,outside,zoo,zoo,25.777390810890576,4.81,1804
+English (American),butt,nouns,body_parts,buttocks/bottom*,bottom,21.98056796070707,4.75,2253
+English (American),did,function_words,helping_predicates,did/did ya,did ya,28.236905169713484,2.45,195
+English (American),in,function_words,locations,inside/in,in,24.07962150691501,3,276036
+English (American),need,function_words,helping_predicates,need/need to,need to,28.517326473959855,1.69,829
+English (American),soda,nouns,food_drink,soda/pop,pop,24.87205183714649,4.97,2512
+English (American),tissue,nouns,household,tissue/kleenex,kleenex,24.881517554334437,4.93,91
+English (American),try,function_words,helping_predicates,try/try to,try to,28.751119300000134,2.22,1273
+English (American),wanna,function_words,helping_predicates,wanna/want to,want to,25.18163215638973,1.93,1415

evaluation-pipeline/babylm_eval.py

@@ -0,0 +1,97 @@
+import argparse
+import lm_eval
+import os
+import json
+
+TASKS = {
+    "blimp": ["anaphor_agreement.json", "argument_structure.json", "binding.json",
+              "control_raising.json", "determiner_noun_agreement.json", "ellipsis.json",
+              "filler_gap.json", "irregular_forms.json", "island_effects.json",
+              "npi_licensing.json", "quantifiers.json", "subject_verb_agreement.json"],
+    "supplement": ["hypernym.json", "qa_congruence_easy.json", "qa_congruence_tricky.json",
+               "subject_aux_inversion.json", "turn_taking.json"]
+}
+
+
+def accuracy_on_task(task_name, eval_model, template_name, num_fewshot):
+    predictions_path = os.path.join(args.model_path, "zeroshot", task_title, "predictions.txt")
+    predictions_dir = os.path.dirname(predictions_path)
+    if not os.path.exists(predictions_dir):
+        os.makedirs(predictions_dir)
+
+    eval_task = lm_eval.get_task_list(task_name, template_names=[template_name])
+    results = lm_eval.evaluate(model=eval_model, tasks=eval_task, seed=12,
+                               num_fewshot=num_fewshot, predictions_path=predictions_path)
+    accuracy = results['results'][0]['acc']
+    return accuracy
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("model_path", type=str,
+                        help="Path to huggingface model and tokenizer.")
+    parser.add_argument("model_type", type=str, choices=["decoder only", "decoder", "encoder only", "encoder", "encoder-decoder",],
+                        help="Language model architecture.")
+    parser.add_argument("--tasks", "-t", type=str, choices=["blimp", "supplement", "all"], default="all",
+                        help="Tasks on which we evaluate.")
+    parser.add_argument("--run_aoa", "-a", action="store_true",
+                        help="Will run the additional AoA prediction task.")
+    parser.add_argument("--trust_remote_code", "-r", action="store_true",
+                        help="Trust remote code (e.g. from huggingface) when loading model.")
+    parser.add_argument("--num_fewshot", "-n", type=int, default=0,
+                        help="Number of few-shot examples to show the model for each test example.")
+    args = parser.parse_args()
+
+    MODEL_TYPE_REMAP = {"decoder only": "hf-causal", "decoder": "hf-causal",
+                        "encoder only": "hf-mlm", "encoder": "hf-mlm",
+                        "encoder-decoder": "hf-seq2seq",}
+    eval_model = lm_eval.get_model(MODEL_TYPE_REMAP[args.model_type],
+                                   pretrained=args.model_path,
+                                   trust_remote_code=args.trust_remote_code,
+                                   device="cuda")
+    tasks = []
+    if args.tasks == "all":
+        for task_type in TASKS.keys():
+            tasks.extend(TASKS[task_type])
+    else:
+        tasks = TASKS[args.tasks]
+
+    accuracies = {}
+    # Iterate through tasks, get accuracies
+    for task in tasks:
+        if task in TASKS["blimp"]:
+            template = None
+            task_title = task.split(".json")[0]
+            task = f"blimp_from_file:filter-data/blimp_filtered/{task}"
+        elif task in TASKS["supplement"]:
+            template = None
+            task_title = task.split(".json")[0]
+            task = f"blimp_from_file:filter-data/supplement_filtered/{task}"
+        else:
+            raise ValueError("Unrecognized task!")
+        accuracies[task_title] = accuracy_on_task(task, eval_model, template,
+                    args.num_fewshot)
+        print(f"{task_title}:\t{accuracies[task_title] * 100:.2f}%")
+        # Write scores to file
+        out_path = os.path.join(args.model_path, "zeroshot", task_title, "eval_results.json")
+        out_dir = os.path.dirname(out_path)
+        if not os.path.exists(out_dir):
+            os.makedirs(out_dir)
+        with open(out_path, 'w') as out_file:
+            json.dump({"eval_accuracy": accuracies[task_title]}, out_file)
+
+    # Print scores
+    print("\nScores:")
+    for task in accuracies.keys():
+        print(f"{task}:\t{accuracies[task] * 100:.2f}%")
+
+    if args.run_aoa:
+        # Run AoA prediction evaluation
+        word_surprisals_n, mad_results = lm_eval.aoa_pred_eval(eval_model.model, eval_model.tokenizer, MODEL_TYPE_REMAP[args.model_type], batch_size = 32)
+        out_dir = os.path.join(args.model_path, "aoa_prediction")
+        if not os.path.exists(out_dir):
+            os.makedirs(out_dir)
+        with open(os.path.join(out_dir, "extracted_average_surprisals.json") , 'w') as out_file:
+            json.dump(word_surprisals_n, out_file)
+        with open(os.path.join(out_dir, "mean_absolute_deviation_results.json"), 'w') as out_file:
+            json.dump(mad_results, out_file)

evaluation-pipeline/collect_results.py

@@ -0,0 +1,85 @@
+import argparse
+import os
+import json
+
+TASKS = {
+    "glue": ["cola", "sst2", "mrpc", "qqp", "mnli", "mnli-mm", "qnli", "rte",
+             "boolq", "multirc", "wsc"],
+    "blimp": ["anaphor_agreement", "argument_structure", "binding", "control_raising",
+              "determiner_noun_agreement", "ellipsis", "filler_gap", "irregular_forms",
+              "island_effects", "npi_licensing", "quantifiers", "subject_verb_agreement"],
+    "supplement": ["hypernym", "qa_congruence_easy", "qa_congruence_tricky",
+                   "subject_aux_inversion", "turn_taking"],
+    "msgs": ["main_verb_control", "control_raising_control", "syntactic_category_control",
+             "relative_position_control", "lexical_content_the_control",
+             "main_verb_lexical_content_the", "main_verb_relative_token_position",
+             "control_raising_lexical_content_the", "control_raising_relative_token_position",
+             "syntactic_category_lexical_content_the", "syntactic_category_relative_position"]
+}
+
+def make_task_dict(task_name, preds_path):
+    def _add_to_dict(index, prediction, task_dict):
+        example_id = f"{task_name}_{index}"
+        prediction = prediction.replace("\\n", "\n")
+        task_dict["predictions"].append({"id": example_id, "pred": prediction})
+
+    if task_name in TASKS["glue"]:
+        task_type = "glue"
+    elif task_name in TASKS["blimp"]:
+        task_type = "blimp"
+    elif task_name in TASKS["supplement"]:
+        task_type = "supplement"
+    elif task_name in TASKS["msgs"]:
+        task_type = "msgs"
+    else:
+        raise ValueError(f"Invalid task: {task_name}!")
+
+    if not os.path.exists(preds_path):
+        raise FileNotFoundError(f"Warning: no predictions found for the \"{task_name}\" ({task_type}) task!")
+
+    task_dict = {"task": task_type, "sub_task": task_name, "predictions": []}
+    with open(preds_path, 'r') as predictions_file:
+        # skip header
+        next(predictions_file)
+        # collect predictions with ids
+        index = None
+        prediction = None
+        for line in predictions_file:
+            if "\t" in line:
+                # add to prediction list
+                if prediction:
+                    _add_to_dict(index, prediction, task_dict)
+                # start new prediction
+                index, prediction = line.strip().split("\t")
+            else:
+                prediction += "\n" + line.strip()
+        # handle final prediction
+        _add_to_dict(index, prediction, task_dict)
+
+    return task_dict
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("model_path", type=str,
+                        help="Path to huggingface model and tokenizer.")
+    args = parser.parse_args()
+
+    task_dicts = {}
+    for task in TASKS["glue"]:
+        preds_path = os.path.join(args.model_path, "finetune", task, "predict_results.txt")
+        task_dicts[task] = make_task_dict(task, preds_path)
+    for task in TASKS["msgs"]:
+        preds_path = os.path.join(args.model_path, "finetune", task, "predict_results.txt")
+        task_dicts[task] = make_task_dict(task, preds_path)
+    for task in TASKS["blimp"]:
+        preds_path = os.path.join(args.model_path, "zeroshot", task, "predictions.txt")
+        task_dicts[task] = make_task_dict(task, preds_path)
+    for task in TASKS["supplement"]:
+        preds_path = os.path.join(args.model_path, "zeroshot", task, "predictions.txt")
+        task_dicts[task] = make_task_dict(task, preds_path)
+
+    with open("all_predictions.json", "w") as predictions_out:
+        for task in task_dicts:
+            predictions_out.write(json.dumps(task_dicts[task]) + "\n")
+    print("Predictions output at `all_predictions.json`.")

evaluation-pipeline/docs/task_guide.md

@@ -0,0 +1,300 @@
+# `Task` Guide
+
+The `Task` class is the foundation of all natural language tasks in the `lm-evaluation-harness` (harness). It encompasses everything you’d need to perform few-shot evaluation of an autoregressive language model. Here we’ll provide a step-by-step guide on how to subclass `Task` to create your very own task/s.
+
+## Setup
+
+If you haven't already, go ahead and fork the main repo, clone it, create a branch with the name of your task, and install the project requirements in your environment:
+
+```sh
+# After forking...
+git clone https://github.com/<YOUR-USERNAME>/lm-evaluation-harness.git
+cd lm-evaluation-harness
+git checkout -b <task-name>
+pip install -e ".[dev]"
+```
+
+## Creating Your Task File
+
+From the `lm-evaluation-harness` project root, copy over the `new_task.py` template to `lm_eval/datasets`.
+
+```sh
+cp templates/new_task.py lm_eval/tasks/<task-name>.py
+```
+
+or if your task is **multiple-choice**, the `new_multiple_choice_task.py`:
+
+```sh
+cp templates/new_multiple_choice_task.py lm_eval/tasks/<task-name>.py
+```
+
+This will set you up with a few `TODO`s to fill-in which we'll now go over in detail.
+
+## Task Heading
+
+Open the file you've just created and add a multiline docstring on the first line with the following contents:
+
+```python
+"""
+<Paper title>
+<Paper PDF URL>
+
+<Short description of task>
+
+Homepage: <URL to task's homepage>
+"""
+```
+
+For example, take the QuAC dataset. We have:
+
+```python
+"""
+QuAC: Question Answering in Context
+https://arxiv.org/abs/1808.07036
+
+Question Answering in Context (QuAC) is a dataset for modeling, understanding, and
+participating in information seeking dialog. Data instances consist of an interactive
+dialog between two crowd workers: (1) a student who poses a sequence of freeform
+questions to learn as much as possible about a hidden Wikipedia text, and (2)
+a teacher who answers the questions by providing short excerpts (spans) from the text.
+
+Homepage: https://quac.ai/
+"""
+```
+
+Next, at the module-level, create a constant variable named
+`_CITATION` that contains the citation information for your task in BibTeX format.
+
+Now let's walk through the actual implementation - from data handling to evaluation.
+
+## Data Handling
+
+### Downloading your Data
+
+All data downloading and management is handled through the HuggingFace (**HF**) [`datasets`](https://github.com/huggingface/datasets) API. So, the first thing you should do is check to see if your task's dataset is already provided in their catalog [here](https://huggingface.co/datasets). If it's not in there, please consider adding it to their Hub to make it accessible to a wider user base by following their [new dataset guide](https://github.com/huggingface/datasets/blob/master/ADD_NEW_DATASET.md)
+.
+Now, that you have your HF dataset, you need to assign its path and name to your `Task` in the following fields:
+
+```python
+class TaskName(...):
+    DATASET_PATH = "..."
+    DATASET_NAME = "..."
+```
+
+where `DATASET_PATH` is the name of the dataset as listed by HF in the `datasets` Hub and `DATASET_NAME` is the name of, what HF calls, a “data instance” or sub-task of the benchmark. If your task does not contain any data instances, just set `DATASET_NAME = None`.
+(If you're familiar with the HF `datasets.load_dataset` function, these are just the first 2 arguments to it.)
+
+Next up, we have to set some “flags”:
+
+```python
+    def has_training_docs(self):
+        return # True/False
+
+    def has_validation_docs(self):
+        return # True/False
+
+    def has_test_docs(self):
+        return # True/False
+```
+
+These methods return `True`/`False` whether or not your task dataset provides documents for each split type. __Note__: if the test set does not have publicly available answer labels, please do not put it down as having a test set - return False.
+
+Lastly, we need to load the documents. In our terminology, a document (`doc`) is a single natural language data example stored in a Python `dict`. E.g.: `{“question”: “What is the capital of France?”, “answer”: “Paris”}`. Override the following methods to load your data splits from their storage location in `DATASET_PATH`:
+
+```python
+    def training_docs(self):
+        return #...
+
+    def validation_docs(self):
+        return #...
+
+    def test_docs(self):
+        return #...
+```
+
+These should return a Python iterable (`list` or `generator`) of `dict`s that can be queried for individual `doc` examples.
+
+#### Processing Documents
+
+At this point, you can also process each individual document to, for example, strip whitespace or "detokenize" its fields. Put the processing logic into `_process_doc` and map the functions across training/validation/test docs inside of the respective functions.
+🔠 If your task is **multiple-choice**, we require you to format your documents such that they contain `gold` and `choices` fields. They can also have other fields, but those will be ignored by `MultipleChoiceTask`. `choices` should be a list of possible continuations, and `gold` should be an integer specifying the index of the correct completion.
+See [this task](https://github.com/EleutherAI/lm-evaluation-harness/blob/6caa0afd96a7a7efb2ec4c1f24ad1756e48f3aa7/lm_eval/tasks/sat.py#L60) for an example. 🔠
+
+### Formatting your Few-Shot Examples
+
+The harness is designed to facilitate task evaluations under the few-shot setting. Here we’ll format such examples.
+
+Format your document into a single query prompt __without the answer__ here. This method takes a single `doc` example of type `dict` with `str` key-value members. You should concatenate these `doc` item values together into a neatly formatted prompt.
+
+```python
+def doc_to_text(self, doc):
+    return ""
+```
+
+<br>
+
+️🔠 **Multiple-Choice Formatting**
+
+If your task is multiple-choice, you can now skip ahead to <a href="#Registering-Your-Task">registering your task</a>.
+
+️️🔠 **End Multiple-Choice Formatting**
+
+<br>
+
+Format the target answer from the contents of `doc`. Note that the prepended `" "` is required to space out the `doc_to_text` and `doc_to_target` strings.
+
+```python
+def doc_to_target(self, doc):
+    target = ""
+    return " " + target
+```
+
+Finally, be aware that the strings from `doc_to_text` and `doc_to_target` will be concatenated together to build up labeled examples in the k-shot setting where k > 0. Design with that in mind 👍.
+
+### Registering Your Task
+
+Now's a good time to register your task to expose it for usage. All you'll need to do is import your task module in `lm_eval/tasks/__init__.py` and provide an entry in the `TASK_REGISTRY`  dictionary with the key as the name of your benchmark task (in the form it'll be referred to in the command line) and the value as the task class. See how it's done for other tasks in the [file](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/__init__.py).
+
+### Checking the Data
+
+After registering your task, you can now check on your data downloading and verify that the few-shot samples look as intended. Run the following command with your desired args:
+
+```bash
+python -m scripts.write_out \
+    --output_base_path <path> \
+    --tasks <your-task> \
+    --sets <train | val | test> \
+    --num_fewshot K \
+    --num_examples N \
+```
+
+Open the file specified at the `--output_base_path <path>` and ensure it passes
+a simple eye test.
+
+## Evaluation
+
+**🛑**  If your task is a single-true multiple-choice task and you've correctly inherited from `MultipleChoiceTask` then your job here is done; <a href="#Checking-the-Task-Performance">go ‘head and check on the task performance!</a> 🛑
+
+Now comes evaluation. The methods you'll need to implement are:
+
+```python
+def construct_requests(self, doc, ctx):
+    """Uses RequestFactory to construct Requests and returns an iterable of
+    Requests which will be sent to the LM.
+
+    Args:
+        doc (dict):
+            The document as returned from training_docs, validation_docs, or
+            test_docs.
+        ctx (str):
+            The context string, generated by fewshot_context. This includes
+            the natural language description, as well as the few shot examples,
+            and the question part of the document for `doc`.
+        args (dict):
+            The specifics of the context, including number of few shots.
+
+    Returns:
+        An iterable of `Request` objects.
+    """
+    return ...
+```
+If your task requires generating text you'll need to return a `rf.greedy_until` request otherwise an `rf.loglikelihood` across all labels in a classification tasks will do.
+
+```python
+def process_results(self, doc, results):
+    """Take a single document and the LM results and evaluates, returning a
+    dict where keys are the names of sub-metrics and values are the values of
+    the metric for that one document.
+
+    NOTE: This function automates processing by using the `promptsource`
+    metadata to determine the metric.
+
+    Args:
+        doc (dict):
+            The document as returned from training_docs, validation_docs, or
+            test_docs.
+        results (list):
+            The results of the requests created in construct_requests.
+
+    Returns:
+        A dict of metric results.
+    """
+    return {}
+```
+
+```python
+def aggregation(self):
+    """
+    Returns:
+        A dictionary where keys are the names of sub-metrics and values are
+        functions that aggregate a list of metric scores.
+        {str: [metric_score] -> float}
+    """
+    return {}
+```
+
+See `lm_eval/metrics.py` for a few "built-in" aggregate metrics you can easily import.
+
+```python
+def higher_is_better(self):
+    """
+    Returns:
+        A dictionary where keys are the names of sub-metrics and values are
+        whether a higher value of the sub-metric is better.
+        {str: bool}
+    """
+    return {}
+```
+
+Some tasks that are good examples of various ways evaluation can be implemented can be found here: [LAMBADA](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/lambada.py), [TriviaQA](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/triviaqa.py), [SQuAD](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/squad.py).
+
+Tip: Feel free to create your own helper-methods for your task!
+
+### Checking the Task Performance
+
+```sh
+python main.py \
+	--model gpt2 \
+	--model_args device=<device-name> \
+	--tasks <task-name> \
+	--num_fewshot K
+```
+
+Set the limit size, `N`, to a smallish number (e.g. 10) and try out the task under different `K`-shot settings. If you have an Nvidia GPU at your disposal, add the argument
+`--model_args device=cuda:0`. If you have access to an OpenAI API key, you can also evaluate GPT-3 on various tasks with the following command:
+
+```sh
+export OPENAI_API_SECRET_KEY=YOUR_KEY_HERE
+python main.py \
+	--model gpt3 \
+	--tasks <task-name> \
+	--num_fewshot K
+```
+
+### Running Unit Tests
+
+To run the entire test suite, use:
+
+```sh
+pytest
+```
+
+This is usually overkill; to run only the tests for your task, do:
+```sh
+pytest -k <task name>
+```
+
+## Versioning
+
+Lastly, we need to "version control". Tasks in the harness can always evolve. Metrics get updated, data sources change, etc. It’s important to mark each task with a version attribute so users can document which implementation version was used to obtain their results. Add a `VERSION` attribute to your task right below the class name and set it to `0` (this is the first version/implementation of your task):
+
+```python
+class TaskName(...):
+	VERSION = 0
+```
+
+## Submitting your Task
+
+Although we currently do not work behind a specific style guide, we'd appreciate if you tidy up your file/s with the `black` formatter (which should've been install through the `requirements.txt`). Keep things clean…ish 🙂.
+
+Now push your work and make a pull request! Thanks for the contribution 👍. If there are any questions, leave a message in the `#lm-thunderdome` channel on the EAI discord.

evaluation-pipeline/filter_data.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8536705c9bb432cc84a9746fe19dacea457b7e2280602fab01431167123dd0b8
+size 61762326

evaluation-pipeline/finetune_all_tasks.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+MODEL_PATH=$1
+LR=${2:-5e-5}
+PATIENCE=${3:-10}
+BSZ=${4:-64}
+EVAL_EVERY=${5:-200}
+MAX_EPOCHS=${6:-10}
+SEED=${7:-12}
+
+# Fine-tune and evaluate on (Super)GLUE tasks
+# If your system uses sbatch or qsub, consider using that to parallelize calls to finetune_model.sh
+for subtask in {"cola","sst2","mrpc","qqp","mnli","mnli-mm","qnli","rte","boolq","multirc","wsc"}; do
+    ./finetune_model.sh $MODEL_PATH glue $subtask $LR $PATIENCE $BSZ $EVAL_EVERY $MAX_EPOCHS $SEED
+done
+
+# Fine-tune and evaluate on MSGS tasks
+for subtask in {"main_verb_control","control_raising_control","syntactic_category_control","lexical_content_the_control","relative_position_control","main_verb_lexical_content_the","main_verb_relative_token_position","syntactic_category_lexical_content_the","syntactic_category_relative_position","control_raising_lexical_content_the","control_raising_relative_token_position"}; do
+	./finetune_model.sh $MODEL_PATH msgs $subtask $LR $PATIENCE $BSZ $EVAL_EVERY $MAX_EPOCHS $SEED
+done

evaluation-pipeline/finetune_classification.py

@@ -0,0 +1,728 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE."""
+# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
+
+import logging
+import os
+import random
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+import evaluate
+
+import datasets
+import numpy as np
+from datasets import load_dataset
+from sklearn.metrics import f1_score
+
+import transformers
+from transformers import (
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    DataCollatorWithPadding,
+    EarlyStoppingCallback,
+    EvalPrediction,
+    HfArgumentParser,
+    IntervalStrategy,
+    PretrainedConfig,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    set_seed,
+)
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils.versions import require_version
+
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+# check_min_version("4.27.0.dev0")
+
+require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt")
+
+task_to_keys = {
+    "cola": ("sentence", None),
+    "mnli": ("premise", "hypothesis"),
+    "mrpc": ("sentence1", "sentence2"),
+    "qnli": ("question", "sentence"),
+    "qqp": ("question1", "question2"),
+    "rte": ("sentence1", "sentence2"),
+    "sst2": ("sentence", None),
+    "stsb": ("sentence1", "sentence2"),
+    "wnli": ("sentence1", "sentence2"),
+}
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+
+    Using `HfArgumentParser` we can turn this class
+    into argparse arguments to be able to specify them on
+    the command line.
+    """
+
+    task_name: Optional[str] = field(
+        default=None,
+        metadata={"help": "The name of the task to train on: " + ", ".join(task_to_keys.keys())},
+    )
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    max_seq_length: int = field(
+        default=128,
+        metadata={
+            "help": (
+                "The maximum total input sequence length after tokenization. Sequences longer "
+                "than this will be truncated, sequences shorter will be padded."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
+    )
+    pad_to_max_length: bool = field(
+        default=True,
+        metadata={
+            "help": (
+                "Whether to pad all samples to `max_seq_length`. "
+                "If False, will pad the samples dynamically when batching to the maximum length in the batch."
+            )
+        },
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_predict_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of prediction examples to this "
+                "value if set."
+            )
+        },
+    )
+    patience: Optional[int] = field(
+       default=None,
+       metadata={
+            "help": (
+                "Number of evaluation steps without improvement > epsilon before stopping fine-tuning. "
+                "Requires the use of the --eval_every argument."
+            )
+       },
+    )
+    eval_every: Optional[int] = field(
+        default=None,
+        metadata = {
+            "help": (
+                "Number of steps between evaluations (MUST be set if patience is set)."
+            )
+        }
+    )
+    train_file: Optional[str] = field(
+        default=None, metadata={"help": "A csv or a json file containing the training data."}
+    )
+    validation_file: Optional[str] = field(
+        default=None, metadata={"help": "A csv or a json file containing the validation data."}
+    )
+    test_file: Optional[str] = field(default=None, metadata={"help": "A csv or a json file containing the test data."})
+
+    def __post_init__(self):
+        if self.task_name is not None:
+            self.task_name = self.task_name.lower()
+            if self.task_name not in task_to_keys.keys():
+                raise ValueError("Unknown task, you should pick one in " + ",".join(task_to_keys.keys()))
+        elif self.dataset_name is not None:
+            pass
+        elif self.train_file is None or self.validation_file is None:
+            raise ValueError("Need either a GLUE task, a training/validation file or a dataset name.")
+        else:
+            train_extension = self.train_file.split(".")[-1]
+            assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file."
+            validation_extension = self.validation_file.split(".")[-1]
+            assert (
+                validation_extension == train_extension
+            ), "`validation_file` should have the same extension (csv or json) as `train_file`."
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    freeze_model: bool = field(
+        default=False,
+        metadata={"help": "Whether to freeze the parameters of the base model."}
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    use_auth_token: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
+                "with private models)."
+            )
+        },
+    )
+    ignore_mismatched_sizes: bool = field(
+        default=False,
+        metadata={"help": "Will enable to load a pretrained model whose head dimensions are different."},
+    )
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+    
+    # Check for the use of early stopping
+    if data_args.patience: 
+        training_args.eval_steps = data_args.eval_every
+        training_args.save_total_limit = 1
+        training_args.load_best_model_at_end = True
+        training_args.evaluation_strategy = "steps"
+        callbacks = [EarlyStoppingCallback(early_stopping_patience=data_args.patience,
+                                           early_stopping_threshold=0.001)]
+    else:
+        callbacks = None
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_glue", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below)
+    # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the
+    # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named
+    # label if at least two columns are provided.
+    #
+    # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this
+    # single column. You can easily tweak this behavior (see below)
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.task_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            "glue",
+            data_args.task_name,
+            cache_dir=model_args.cache_dir,
+            use_auth_token=True if model_args.use_auth_token else None,
+        )
+    elif data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            use_auth_token=True if model_args.use_auth_token else None,
+        )
+    else:
+        # Loading a dataset from your local files.
+        # CSV/JSON training and evaluation files are needed.
+        data_files = {"train": data_args.train_file, "validation": data_args.validation_file}
+
+        # Get the test dataset: you can provide your own CSV/JSON test file (see below)
+        # when you use `do_predict` without specifying a GLUE benchmark task.
+        if training_args.do_predict:
+            if data_args.test_file is not None:
+                train_extension = data_args.train_file.split(".")[-1]
+                test_extension = data_args.test_file.split(".")[-1]
+                assert (
+                    test_extension == train_extension
+                ), "`test_file` should have the same extension (csv or json) as `train_file`."
+                data_files["test"] = data_args.test_file
+            else:
+                train_extension = data_args.train_file.split(".")[-1]
+                validation_extension = data_args.validation_file.split(".")[-1]
+                assert (
+                    validation_extension == train_extension
+                ), "`validation_file` should have the same extension (csv or json) as `train_file`."
+                data_files["test"] = data_args.validation_file
+
+        for key in data_files.keys():
+            logger.info(f"load a local file for {key}: {data_files[key]}")
+
+        if data_args.train_file.endswith(".csv"):
+            # Loading a dataset from local csv files
+            raw_datasets = load_dataset(
+                "csv",
+                data_files=data_files,
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+            )
+        else:
+            # Loading a dataset from local json files
+            raw_datasets = load_dataset(
+                "json",
+                data_files=data_files,
+                cache_dir=model_args.cache_dir,
+                use_auth_token=True if model_args.use_auth_token else None,
+            )
+    # See more about loading any type of standard or custom dataset at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    # Labels
+    if data_args.task_name is not None:
+        is_regression = data_args.task_name == "stsb"
+        if not is_regression:
+            label_list = raw_datasets["train"].features["label"].names
+            num_labels = len(label_list)
+        else:
+            num_labels = 1
+    else:
+        # Trying to have good defaults here, don't hesitate to tweak to your needs.
+        is_regression = raw_datasets["train"].features["label"].dtype in ["float32", "float64"]
+        if is_regression:
+            num_labels = 1
+        else:
+            # A useful fast method:
+            # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique
+            label_list = raw_datasets["train"].unique("label")
+            label_list.sort()  # Let's sort it for determinism
+            num_labels = len(label_list)
+    
+    is_binary = (num_labels == 2)
+
+    # Load pretrained model and tokenizer
+    #
+    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+    config = AutoConfig.from_pretrained(
+        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=data_args.task_name,
+        cache_dir=model_args.cache_dir,
+        revision=model_args.model_revision,
+        use_auth_token=True if model_args.use_auth_token else None,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        use_fast=model_args.use_fast_tokenizer,
+        revision=model_args.model_revision,
+        use_auth_token=True if model_args.use_auth_token else None,
+    )
+    try:
+        model = AutoModelForSequenceClassification.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            use_auth_token=True if model_args.use_auth_token else None,
+            ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,
+        )
+    except ValueError as e:
+        from transformers import T5Config
+        if isinstance(config, T5Config):
+            from transformers_modified.t5 import T5ForSequenceClassification
+            model = T5ForSequenceClassification.from_pretrained(
+                model_args.model_name_or_path,
+                from_tf=bool(".ckpt" in model_args.model_name_or_path),
+                config=config,
+                cache_dir=model_args.cache_dir,
+                revision=model_args.model_revision,
+                use_auth_token=True if model_args.use_auth_token else None,
+                ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,
+            )
+        else:
+            raise e
+    
+    # Freeze all parameters (including embeddings) except the classifier head
+    if model_args.freeze_model:
+        for name, param in model.named_parameters():
+            if "classifier" not in name and not name.startswith("score"): # classifier layer
+                param.requires_grad = False
+
+    # Preprocessing the raw_datasets
+    template = None
+    if data_args.task_name is not None:
+        sentence1_key, sentence2_key = task_to_keys[data_args.task_name]
+    else:
+        # Again, we try to have some nice defaults but don't hesitate to tweak to your use case.
+        non_label_column_names = [name for name in raw_datasets["train"].column_names if name not in ("label", "idx")]
+        if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names:
+            sentence1_key, sentence2_key = "sentence1", "sentence2"
+        elif "question1" in non_label_column_names and "question2" in non_label_column_names:
+            sentence1_key, sentence2_key = "question1", "question2"
+        elif "premise" in non_label_column_names and "hypothesis" in non_label_column_names:
+            sentence1_key, sentence2_key = "premise", "hypothesis"
+        elif "question" in non_label_column_names and "passage" in non_label_column_names:
+            sentence1_key, sentence2_key = "question", "passage"
+        elif "question" in non_label_column_names and "sentence" in non_label_column_names:
+            sentence1_key, sentence2_key = "question", "sentence"
+        # special cases
+        elif "paragraph" in non_label_column_names and \
+             "question" in non_label_column_names and \
+             "answer" in non_label_column_names:        # MultiRC
+            sentence1_key, sentence2_key = ["question", "answer"], "paragraph"
+            template = "Question: {} Answer: {}"
+        elif "text" in non_label_column_names and \
+             "span1_text" in non_label_column_names and \
+             "span2_text" in non_label_column_names:    # WSC
+            sentence1_key, sentence2_key = ["span2_text", "span1_text"], "text"
+            template = "Does \"{}\" refer to \"{}\" in this passage?"
+        elif "sentence" in non_label_column_names and "linguistic_feature_type" in non_label_column_names:
+            sentence1_key, sentence2_key = "sentence", None
+        else:
+            if len(non_label_column_names) >= 2:
+                sentence1_key, sentence2_key = non_label_column_names[:2]
+            else:
+                sentence1_key, sentence2_key = non_label_column_names[0], None
+
+    # Padding strategy
+    if data_args.pad_to_max_length:
+        padding = "max_length"
+    else:
+        # We will pad later, dynamically at batch creation, to the max sequence length in each batch
+        padding = False
+
+    # Some models have set the order of the labels to use, so let's make sure we do use it.
+    label_to_id = None
+    if (
+        model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id
+        and data_args.task_name is not None
+        and not is_regression
+    ):
+        # Some have all caps in their config, some don't.
+        label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()}
+        if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)):
+            label_to_id = {i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)}
+        else:
+            logger.warning(
+                "Your model seems to have been trained with labels, but they don't match the dataset: ",
+                f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}."
+                "\nIgnoring the model labels as a result.",
+            )
+    elif data_args.task_name is None and not is_regression:
+        label_to_id = {v: i for i, v in enumerate(label_list)}
+
+    if label_to_id is not None:
+        model.config.label2id = label_to_id
+        model.config.id2label = {id: label for label, id in config.label2id.items()}
+    elif data_args.task_name is not None and not is_regression:
+        model.config.label2id = {l: i for i, l in enumerate(label_list)}
+        model.config.id2label = {id: label for label, id in config.label2id.items()}
+
+    if data_args.max_seq_length > tokenizer.model_max_length:
+        logger.warning(
+            f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
+            f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
+        )
+    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
+
+    def preprocess_function(examples):
+        # Tokenize the texts
+        if isinstance(sentence1_key, list):
+            keya, keyb = examples[sentence1_key[0]], examples[sentence1_key[1]]
+            keys1 = [template.format(ka, kb) for ka, kb in zip(keya, keyb)]
+            args = (
+                (keys1, examples[sentence2_key])
+            )
+        else:
+            args = (
+                (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key])
+            )
+        result = tokenizer(*args, padding=padding, max_length=max_seq_length, truncation=True)
+
+        # Map labels to IDs (not necessary for GLUE tasks)
+        if label_to_id is not None and "label" in examples:
+            result["label"] = [(label_to_id[l] if l != -1 else -1) for l in examples["label"]]
+        return result
+
+    with training_args.main_process_first(desc="dataset map pre-processing"):
+        raw_datasets = raw_datasets.map(
+            preprocess_function,
+            batched=True,
+            load_from_cache_file=not data_args.overwrite_cache,
+            desc="Running tokenizer on dataset",
+        )
+    if training_args.do_train:
+        if "train" not in raw_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = raw_datasets["train"]
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+
+    if training_args.do_eval:
+        if "validation" not in raw_datasets and "validation_matched" not in raw_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = raw_datasets["validation_matched" if data_args.task_name == "mnli" else "validation"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+
+    if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None:
+        if "test" not in raw_datasets and "test_matched" not in raw_datasets:
+            raise ValueError("--do_predict requires a test dataset")
+        predict_dataset = raw_datasets["test_matched" if data_args.task_name == "mnli" else "test"]
+        if data_args.max_predict_samples is not None:
+            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)
+            predict_dataset = predict_dataset.select(range(max_predict_samples))
+
+    # Log a few random samples from the training set:
+    if training_args.do_train:
+        for index in random.sample(range(len(train_dataset)), 3):
+            logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")
+
+    # Get the metric function
+    if data_args.task_name is not None:
+        metric = evaluate.load("glue", data_args.task_name)
+    else:
+        metric = evaluate.load("accuracy")
+
+    # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
+    # predictions and label_ids field) and has to return a dictionary string to float.
+    def compute_metrics(p: EvalPrediction):
+        preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions
+        preds = np.squeeze(preds) if is_regression else np.argmax(preds, axis=1)
+        if data_args.task_name is not None:
+            result = metric.compute(predictions=preds, references=p.label_ids)
+            if len(result) > 1:
+                result["combined_score"] = np.mean(list(result.values())).item()
+            return result
+        elif is_regression:
+            return {"mse": ((preds - p.label_ids) ** 2).mean().item()}
+        elif is_binary:
+            return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item(),
+                    "f1": f1_score(y_true=p.label_ids, y_pred=preds, average="binary")}
+        else:
+            return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item()}
+
+    # Data collator will default to DataCollatorWithPadding when the tokenizer is passed to Trainer, so we change it if
+    # we already did the padding.
+    if data_args.pad_to_max_length:
+        data_collator = default_data_collator
+    elif training_args.fp16:
+        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
+    else:
+        data_collator = None
+    
+    if is_regression:
+        training_args.metric_for_best_model = "mse"
+    elif is_binary:
+        training_args.metric_for_best_model = "f1"
+        training_args.greater_is_better = True
+    else:
+        training_args.metric_for_best_model = "accuracy"
+        training_args.greater_is_better = True
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        compute_metrics=compute_metrics,
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+        callbacks = callbacks,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        metrics = train_result.metrics
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        # Loop to handle MNLI double evaluation (matched, mis-matched)
+        tasks = [data_args.task_name]
+        eval_datasets = [eval_dataset]
+        if data_args.task_name == "mnli":
+            tasks.append("mnli-mm")
+            valid_mm_dataset = raw_datasets["validation_mismatched"]
+            if data_args.max_eval_samples is not None:
+                max_eval_samples = min(len(valid_mm_dataset), data_args.max_eval_samples)
+                valid_mm_dataset = valid_mm_dataset.select(range(max_eval_samples))
+            eval_datasets.append(valid_mm_dataset)
+            combined = {}
+
+        for eval_dataset, task in zip(eval_datasets, tasks):
+            metrics = trainer.evaluate(eval_dataset=eval_dataset)
+
+            max_eval_samples = (
+                data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+            )
+            metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+
+            if task == "mnli-mm":
+                metrics = {k + "_mm": v for k, v in metrics.items()}
+            if task is not None and "mnli" in task:
+                combined.update(metrics)
+
+            trainer.log_metrics("eval", metrics)
+            trainer.save_metrics("eval", combined if task is not None and "mnli" in task else metrics)
+
+    if training_args.do_predict:
+        logger.info("*** Predict ***")
+
+        # Loop to handle MNLI double evaluation (matched, mis-matched)
+        # We do not use MNLI test data.
+        tasks = [data_args.task_name]
+        if data_args.task_name == "mnli":
+            tasks.append("mnli-mm")
+            predict_datasets.append(raw_datasets["test_mismatched"])
+
+        # Removing the `label` columns because it contains -1 and Trainer won't like that.
+        predict_dataset = predict_dataset.remove_columns("label")
+        predictions = trainer.predict(predict_dataset, metric_key_prefix="predict").predictions
+        predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1)
+
+        output_predict_file = os.path.join(training_args.output_dir, f"predict_results.txt")
+        if trainer.is_world_process_zero():
+            with open(output_predict_file, "w") as writer:
+                logger.info(f"***** Predict results *****")
+                writer.write("index\tprediction\n")
+                for index, item in enumerate(predictions):
+                    if is_regression:
+                        writer.write(f"{index}\t{item:3.3f}\n")
+                    else:
+                        item = label_list[item]
+                        writer.write(f"{index}\t{item}\n")
+
+    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-classification"}
+    if data_args.task_name is not None:
+        kwargs["language"] = "en"
+        kwargs["dataset_tags"] = "glue"
+        kwargs["dataset_args"] = data_args.task_name
+        kwargs["dataset"] = f"GLUE {data_args.task_name.upper()}"
+
+    """
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+    """
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()

evaluation-pipeline/finetune_model.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+MODEL_PATH=$1
+TASK_NAME=$2
+SUBTASK_NAME=$3
+LR=${4:-5e-5}           # default: 5e-5
+PATIENCE=${5:-10}       # default: 10
+BSZ=${6:-64}            # default: 64
+EVAL_EVERY=${7:-200}    # default: 200
+MAX_EPOCHS=${8:-10}     # default: 10
+SEED=${9:-12}           # default: 12
+
+if [[ "$SUBTASK_NAME" = "mnli" ]]; then
+    VALID_NAME="validation_matched"
+    OUT_DIR="mnli"
+elif [[ "$SUBTASK_NAME" = "mnli-mm" ]]; then
+    VALID_NAME="validation_mismatched"
+    SUBTASK_NAME="mnli"
+    OUT_DIR="mnli-mm"
+else
+    VALID_NAME="validation"
+    OUT_DIR=$SUBTASK_NAME
+fi
+
+mkdir -p $MODEL_PATH/finetune/$OUT_DIR/
+
+python finetune_classification.py \
+  --model_name_or_path $MODEL_PATH \
+  --output_dir $MODEL_PATH/finetune/$OUT_DIR/ \
+  --train_file filter-data/${TASK_NAME}_filtered/$SUBTASK_NAME.train.json \
+  --validation_file filter-data/${TASK_NAME}_filtered/$SUBTASK_NAME.$VALID_NAME.json \
+  --do_train \
+  --do_eval \
+  --do_predict \
+  --use_fast_tokenizer False \
+  --max_seq_length 128 \
+  --per_device_train_batch_size $BSZ \
+  --learning_rate $LR \
+  --num_train_epochs $MAX_EPOCHS \
+  --evaluation_strategy steps \
+  --patience $PATIENCE \
+  --eval_every $EVAL_EVERY \
+  --eval_steps $EVAL_EVERY \
+  --save_steps $EVAL_EVERY \
+  --overwrite_output_dir \
+  --seed $SEED

evaluation-pipeline/ignore.txt

@@ -0,0 +1,9 @@
+ROUGE
+rouge
+nin
+ond
+som
+tha
+vie
+FPR
+fpr

evaluation-pipeline/lm_eval/__init__.py

@@ -0,0 +1,3 @@
+from .evaluator import evaluate
+from .models import get_model, list_model_apis
+from .tasks import get_task, get_task_list, list_tasks, get_templates, list_templates, aoa_pred_eval

evaluation-pipeline/lm_eval/api/metric.py

@@ -0,0 +1,371 @@
+import logging
+import math
+import random
+import numpy as np
+import sacrebleu
+import sklearn.metrics
+from collections.abc import Iterable
+from rouge_score import rouge_scorer
+from typing import List, Mapping, Optional
+
+from lm_eval.metrics import sari as sari_impl
+
+
+logger = logging.getLogger(__name__)
+
+
+def mean(arr):
+    return sum(arr) / len(arr)
+
+
+def pop_stddev(arr):
+    mu = mean(arr)
+    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
+
+
+def sample_stddev(arr):
+    mu = mean(arr)
+    if len(arr) == 1:
+        return 0
+    else:
+        return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
+
+
+def mean_stderr(arr):
+    return sample_stddev(arr) / math.sqrt(len(arr))
+
+
+def median(arr):
+    return arr[len(arr) // 2]
+
+
+def matthews_corrcoef(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    return sklearn.metrics.matthews_corrcoef(golds, preds)
+
+
+def f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = sklearn.metrics.f1_score(golds, preds)
+    return np.max(fscore)
+
+
+def acc_all(items):
+    # Only count as correct if all answers are labeled correctly for each question
+    question_scoring_dict = {}
+    preds = list(zip(*items))[0]
+    docs = list(zip(*items))[1]
+
+    for doc, pred in zip(docs, preds):
+        paragraph_id = doc["idx"]["paragraph"]
+        question_id = doc["idx"]["question"]
+        if (paragraph_id, question_id) not in question_scoring_dict:
+            question_scoring_dict[(paragraph_id, question_id)] = []
+
+        gold_label = doc["label"] == 1
+
+        question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred)
+    acc = np.mean([int(all(x)) for x in question_scoring_dict.values()])
+    return acc
+
+
+def acc_all_stderr(items):
+    # Only count as correct if all answers are labeled correctly for each question
+    question_scoring_dict = {}
+    preds = list(zip(*items))[0]
+    docs = list(zip(*items))[1]
+
+    for doc, pred in zip(docs, preds):
+        question_id = doc["idx"]["question"]
+        if question_id not in question_scoring_dict:
+            question_scoring_dict[question_id] = []
+
+        gold_label = doc["label"] == 1
+        question_scoring_dict[question_id].append(gold_label == pred)
+
+    acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()])
+    return acc
+
+
+def compute_parity_scores(items):
+    # Parity checks whether predictions in subsequent pairs of examples are consistent.
+    # In WinogenderSchema those examples differ only in the gender of the pronoun in the hypothesis.
+    indices2predictions = {idx: pred for idx, pred in items}
+    parity_scores = []
+    for idx in indices2predictions.keys():
+        if (idx % 2) == 0 and (idx + 1) in indices2predictions:
+            parity_scores.append(
+                int(indices2predictions[idx] == indices2predictions[idx + 1])
+            )
+    return parity_scores
+
+
+def parity(items):
+    parity_scores = compute_parity_scores(items)
+    if len(parity_scores) > 0:
+        acc = mean(parity_scores)
+    else:
+        acc = 0.0
+    return acc
+
+
+def parity_stderr(items):
+    parity_scores = compute_parity_scores(items)
+    if len(parity_scores) > 0:
+        stderr = mean_stderr(parity_scores)
+    else:
+        stderr = 0.0
+    return stderr
+
+
+def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
+    """Compute max metric between prediction and each ground truth."""
+    scores_for_ground_truths = []
+    for ground_truth in ground_truths:
+        score = metric_fn(prediction, ground_truth)
+        scores_for_ground_truths.append(score)
+    return max(scores_for_ground_truths)
+
+
+def perplexity(items):
+    return math.exp(-mean(items))
+
+
+def weighted_mean(items):
+    a, b = zip(*items)
+    return sum(a) / sum(b)
+
+
+def weighted_perplexity(items):
+    return math.exp(-weighted_mean(items))
+
+
+def bits_per_byte(items):
+    return -weighted_mean(items) / math.log(2)
+
+
+def sari(sentence_to_simplifiy, generated_sentence, references):
+    """Implementation of SARI from the authors'."""
+    return sari_impl.SARIsent(sentence_to_simplifiy, generated_sentence, references)
+
+
+def bleu(items):
+    """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric
+    for evaluating a generated sentence to a reference sentence. It counts matching
+    n-grams in the candidate translation to n-grams in the reference text, where
+    1-gram or uni-gram would be each token and a bi-gram comparison would be each
+    word pair. The comparison is made regardless of word order
+    Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/
+    Paper: https://www.aclweb.org/anthology/P02-1040/
+
+    Higher is better
+    """
+    refs = list(zip(*items))[0]
+    preds = list(zip(*items))[1]
+    refs, preds = _sacreformat(refs, preds)
+    return sacrebleu.corpus_bleu(preds, refs).score
+
+
+def chrf(items):
+    """chrF++ is a tool for automatic evaluation of machine translation output
+    based on character n-gram precision and recall enhanced with word n-grams.
+    Source: https://github.com/m-popovic/chrF
+    Paper: https://www.aclweb.org/anthology/W15-3049.pdf
+
+    Higher is better
+    """
+    refs = list(zip(*items))[0]
+    preds = list(zip(*items))[1]
+    refs, preds = _sacreformat(refs, preds)
+    return sacrebleu.corpus_chrf(preds, refs).score
+
+
+def ter(items):
+    """Translation Error Rate is an error metric for machine translation that
+    measures the number of edits required to change a system output into one
+    of the references
+    Source: http://www.cs.umd.edu/~snover/tercom/
+    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf
+
+    Lower is better
+    """
+    refs = list(zip(*items))[0]
+    preds = list(zip(*items))[1]
+    refs, preds = _sacreformat(refs, preds)
+    return sacrebleu.corpus_ter(preds, refs).score
+
+
+def is_non_str_iterable(obj):
+    return isinstance(obj, Iterable) and not isinstance(obj, str)
+
+
+def _sacreformat(refs, preds):
+    """Format refs and preds for sacrebleu corpus calculation. It is very particular"""
+    # Sacrebleu expects (List[str], List[List[str])
+    #   e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...])
+
+    # Note [ref1_stream] is the first reference for each pred.
+    # So lists are size N and (M, N) for N preds and M possible refs for each pred
+    # This is a different order of dimensions that I would expect
+
+    # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds
+    # Must become List[List[str]] with the inner list corresponding to preds
+    if not is_non_str_iterable(refs):
+        refs = list(refs)
+    if not is_non_str_iterable(refs[0]):
+        refs = [[ref] for ref in refs]
+    refs = list(zip(*refs))
+    # Note the number of refs in each ref list much match the number of preds
+
+    # We expect preds to be List[str] or List[List[str]]. Must become List[str]
+    if not is_non_str_iterable(preds):
+        preds = list(preds)
+    if is_non_str_iterable(preds[0]):
+        assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}"
+        preds = [pred[0] for pred in preds]
+
+    return refs, preds
+
+
+def rouge(
+    refs: List[str],
+    pred: str,
+    rouge_types: Optional[List[str]] = ["rouge1", "rouge2", "rougeL", "rougeLsum"],
+) -> Mapping[str, float]:
+    """ROUGE with multi-reference support
+
+    Implementation based on GEM-metrics:
+    https://github.com/GEM-benchmark/GEM-metrics/blob/431a8174bd6b3637e8d6118bfad2983e39e99733/gem_metrics/rouge.py
+
+    Args:
+        refs (List[str]):
+            A `list` of reference `str`s.
+        pred (str):
+            A single prediction `str`s.
+        rouge_types (Optional[List[str]]):
+            An optional list of ROUGE types from the set:
+            ["rouge1", "rouge2", "rougeL", "rougeLsum"]
+
+    Returns:
+        A `dict` of ROUGE scores.
+    """
+
+    # Add newlines between sentences to correctly compute `rougeLsum`.
+    if "rougeLsum" in rouge_types:
+        # TODO: Adapt this to handle languages that do not support sentence endings by `.`.
+        # See GEM-metrics implementation with lang specific `nltk` tokenizers to
+        # split sentences.
+        pred = pred.replace(".", ".\n")
+        refs = [ref.replace(".", ".\n") for ref in refs]
+
+    scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=True)
+    # ROUGE multi-ref jackknifing
+    if len(refs) > 1:
+        cur_scores = [scorer.score(ref, pred) for ref in refs]
+
+        # get best score for all leave-one-out sets
+        best_scores = []
+        for leave in range(len(refs)):
+            cur_scores_leave_one = [
+                cur_scores[s] for s in range(len(refs)) if s != leave
+            ]
+            best_scores.append(
+                {
+                    rouge_type: max(
+                        [s[rouge_type] for s in cur_scores_leave_one],
+                        key=lambda s: s.fmeasure,
+                    )
+                    for rouge_type in rouge_types
+                }
+            )
+        # average the leave-one-out bests to produce the final score
+        score = {
+            rouge_type: rouge_scorer.scoring.Score(
+                np.mean([b[rouge_type].precision for b in best_scores]),
+                np.mean([b[rouge_type].recall for b in best_scores]),
+                np.mean([b[rouge_type].fmeasure for b in best_scores]),
+            )
+            for rouge_type in rouge_types
+        }
+    else:
+        score = scorer.score(refs[0], pred)
+    # convert the named tuples to plain nested dicts
+    score = {
+        rouge_type: {
+            "precision": score[rouge_type].precision,
+            "recall": score[rouge_type].recall,
+            "fmeasure": score[rouge_type].fmeasure,
+        }
+        for rouge_type in rouge_types
+    }
+    return score
+
+
+# Standard Error Utils
+
+
+class _BootstrapInternal:
+    def __init__(self, f, n):
+        self.f = f
+        self.n = n
+
+    def __call__(self, v):
+        i, xs = v
+        rnd = random.Random()
+        rnd.seed(i)
+        res = []
+        for _ in range(self.n):
+            res.append(self.f(rnd.choices(xs, k=len(xs))))
+        return res
+
+
+def bootstrap_stderr(f, xs, iters):
+    import multiprocessing as mp
+
+    pool = mp.Pool(mp.cpu_count())
+    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
+    # equivalent to stderr calculated without Bessel's correction in the stddev.
+    # Unfortunately, I haven't been able to figure out what the right correction is
+    # to make the bootstrap unbiased - I considered multiplying by sqrt(n/(n-1)) but
+    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
+    # Thankfully, shouldn't matter because our samples are usually pretty big.
+    res = []
+    chunk_size = min(1000, iters)
+    from tqdm import tqdm
+
+    logger.info("Bootstrapping for stddev:", f.__name__)
+    for bootstrap in tqdm(
+        pool.imap(
+            _BootstrapInternal(f, chunk_size),
+            [(i, xs) for i in range(iters // chunk_size)],
+        ),
+        total=iters // chunk_size,
+    ):
+        # sample w replacement
+        res.extend(bootstrap)
+
+    pool.close()
+    return sample_stddev(res)
+
+
+def stderr_for_metric(metric, bootstrap_iters):
+    bootstrappable = [
+        median,
+        matthews_corrcoef,
+        f1_score,
+        perplexity,
+        bleu,
+        chrf,
+        ter,
+    ]
+
+    if metric in bootstrappable:
+        return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters)
+
+    stderr = {mean: mean_stderr, acc_all: acc_all_stderr, parity: parity_stderr}
+
+    return stderr.get(metric, None)

evaluation-pipeline/lm_eval/api/model.py

@@ -0,0 +1,454 @@
+import abc
+import hashlib
+import json
+import os
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+from typing import Iterable, List, Optional, Tuple, Union
+from transformers import BatchEncoding
+
+from lm_eval.api import utils
+
+
+class LM(abc.ABC):
+    def __init__(self):
+        self.cache_hook = CacheHook(None)
+
+    @abc.abstractmethod
+    def loglikelihood(
+        self, requests: List[Tuple[str, str]]
+    ) -> List[Tuple[float, bool]]:
+        """Compute log-likelihood of generating a continuation from a context.
+        Downstream tasks should attempt to use loglikelihood instead of other
+        LM calls whenever possible.
+
+        Args:
+            requests (List[Tuple[str, str]]):
+                A list of pairs (context, continuation):
+                context (str):
+                    Context string. Implementations of LM must be able to handle
+                    an empty context string.
+                continuation (str):
+                    The continuation over which log likelihood will be calculated.
+                    If there is a word boundary, the space should be in the
+                    continuation. For example, context="hello" continuation=" world"
+                    is correct.
+
+        Returns:
+            A list of pairs (logprob, isgreedy):
+                logprob (float):
+                    The log probability of `continuation`.
+                isgreedy (bool):
+                    Whether `continuation` would be generated by greedy
+                    sampling from `context`.
+        """
+        pass
+
+    @abc.abstractmethod
+    def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]:
+        """Compute full log-likelihood of a string, with no truncation, for perplexity computation
+        - We will use the full max context length of the model.
+        - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to
+        the max context length.
+        - IMPORTANT: Each document's loglikelihood/perplexity is computed *separately*, unlike other implementations
+          which may simply concatenate multiple documents together.
+        - IMPORTANT: We maximize the amount of context for each prediction. Specifically, for inputs that we break into
+          multiple chunks, the last input will still a full-sized context.
+          Example:
+            Input tokens: [ 0 1 2 3 4 5 6 7 8 9 ]
+            Prefix: EOT
+            Max context length: 4
+            Resulting input/prediction pairs:
+
+                INPUT:  EOT   0   1   2
+                PRED:     0   1   2   3
+
+                INPUT:    3   4   5   6
+                PRED:     4   5   6   7
+
+                INPUT:    5   6   7   8
+                PRED:             8   9
+
+          Observe that:
+            1. Each token is predicted exactly once
+            2. For the last pair, we provide the full context, but only score the last two tokens
+
+        Args:
+            requests (List[Tuple[str, str]]):
+                A list of paired strings.
+                string (str):
+                    String for which we are computing per-token loglikelihood.
+
+        Returns:
+            A list of logprobs on the `continuation`.
+        """
+        pass
+
+    @abc.abstractmethod
+    def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]:
+        """Generate greedily until a stopping sequence or max generation length.
+
+        Args:
+            requests (List[Tuple[str, dict]]):
+                A list of pairs (context, args):
+                context (str):
+                    Context string.
+                args (dict):
+                    A dictionary of generation arguments in the form:
+                    {
+                        stop_sequences: str,
+                        max_generation_length: int,
+                        num_fewshot: int
+                    }
+
+        Returns:
+            A list of strings continuation:
+                continuation: str
+                    The generated continuation.
+        """
+        pass
+
+    def set_cache_hook(self, cache_hook: "CacheHook"):
+        self.cache_hook = cache_hook
+
+
+TokenSequence = Union[List[int], torch.LongTensor, torch.Tensor, BatchEncoding]
+
+
+class TokenLM(LM):
+    """A language model that assumes inputs, and possibly outputs, are
+    tokenized text as opposed to language model APIs that only support
+    string-based input and output systems.
+    """
+
+    @abc.abstractmethod
+    def tok_encode(self, string: str):
+        pass
+
+    @abc.abstractmethod
+    def tok_decode(self, tokens: Iterable[int]) -> List[str]:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def eot_token(self) -> str:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def eot_token_id(self) -> int:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def max_gen_toks(self) -> int:
+        """The maximum number of tokens to generate - not including context."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def max_length(self) -> int:
+        """The maximum sequence length of the model."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def batch_size(self) -> int:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def device(self) -> Union[int, str, torch.device]:
+        pass
+
+    def loglikelihood(
+        self, requests: List[Tuple[str, str]]
+    ) -> List[Tuple[float, bool]]:
+        new_requests = []
+        for context, continuation in requests:
+            if context == "":
+                # End of text as context
+                context_enc = [self.eot_token_id]
+            else:
+                context_enc = self.tok_encode(context)
+            continuation_enc = self.tok_encode(continuation)
+            new_requests.append(
+                ((context, continuation), context_enc, continuation_enc)
+            )
+        return self._loglikelihood_tokens(new_requests)
+
+    def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]:
+        # TODO: Implement caching once we've confirmed the perplexity implementation
+        # TODO: Automatic batch size detection for vectorization
+        loglikelihoods = []
+        for (string,) in tqdm(requests):
+            rolling_token_windows = list(
+                map(
+                    utils.make_disjoint_window,
+                    utils.get_rolling_token_windows(
+                        token_list=self.tok_encode(string),
+                        prefix_token=self.eot_token_id,
+                        max_seq_len=self.max_length,
+                        context_len=1,
+                    ),
+                )
+            )
+            rolling_token_windows = [(None,) + x for x in rolling_token_windows]
+            # TODO: Extract out this call so it only gets called once and
+            # also somehow figure out partial caching for that.
+            string_nll = self._loglikelihood_tokens(
+                rolling_token_windows, disable_tqdm=True
+            )
+            # Discard `is_greedy`
+            string_nll = [x[0] for x in string_nll]
+            string_nll = sum(string_nll)
+            loglikelihoods.append(string_nll)
+        return loglikelihoods
+
+    def _loglikelihood_tokens(
+        self,
+        requests: List[Tuple[Tuple[str, str], TokenSequence, TokenSequence]],
+        disable_tqdm: Optional[bool] = False,
+    ) -> List[Tuple[float, bool]]:
+        """Helper method for computing log-likelihood of generating a
+        continuation from a context that have both been tokenized/encoded.
+
+        Args:
+            requests (List[Tuple[Tuple[str, str], TokenSequence, TokenSequence]]):
+                A list of pairs ((context, continuation), context_enc, continuation_enc):
+                context (str):
+                    Context string. Implementations of LM must be able to handle
+                    an empty context string.
+                continuation (str):
+                    The continuation over which log likelihood will be calculated.
+                    If there is a word boundary, the space should be in the
+                    continuation. For example, context="hello" continuation=" world"
+                    is correct.
+                context_enc (TokenSequence):
+                    The tokenized/encoded context.
+                continuation_enc (TokenSequence):
+                    The tokenized/encoded continuation.
+            disable_tqdm (bool, optional, defaults to False):
+                Whether to disable `tqdm` progress bar.
+
+        Returns:
+            A list of pairs (logprob, isgreedy):
+                logprob (float):
+                    The log probability of `continuation`.
+                isgreedy (float):
+                    Whether `continuation` would be generated by greedy sampling from `context`.
+        """
+
+        def _collate(x):
+            # The negative sign on len(tokens) sorts descending - this has a few advantages:
+            # - Time estimates will always be over not underestimates, which is more useful for planning
+            # - To know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much easier to implement
+            # - Any OOMs will happen right away rather than near the end
+            tokens = x[1] + x[2]
+            return -len(tokens), tuple(tokens)
+
+        # TODO: Automatic (variable) batch size detection for vectorization
+        # TODO: Implement some kind of efficient-request-middleware that lumps together requests with the same context
+        results = []
+        reorder = utils.Reorderer(requests, _collate)
+        for chunk in utils.chunks(
+            tqdm(reorder.get_reordered(), disable=disable_tqdm), self.batch_size
+        ):
+            inputs = []
+            input_lens = []
+            cont_tokens_list = []
+            padding_length = None
+
+            # Because vectorizing is annoying, we first convert each (context, continuation) pair to padded
+            # tensors, then we pack them together into a batch, call the model, and then pick it all apart
+            # again because vectorizing is annoying
+            for _, context_enc, continuation_enc in chunk:
+                # sanity check
+                assert len(context_enc) > 0
+                assert len(continuation_enc) > 0
+                assert len(continuation_enc) <= self.max_length
+
+                # How this all works:
+                #          CTX      CONT
+                # inp    0 1 2 3|4 5 6 7 8 9   <- last token is deleted by inp[:, :-1]
+                # gpt2    \               \
+                # logits   1 2 3|4 5 6 7 8 9   <- the ctx half gets tossed out by the
+                # cont_tokens      4 5 6 7 8 9      [:, -len(continuation_enc):, :self.vocab_size] slice
+
+                # When too long to fit in context, truncate from the left
+                _full_enc = context_enc + continuation_enc
+                input = torch.tensor(
+                    _full_enc[-(self.max_length + 1) :][:-1],
+                    dtype=torch.long,
+                ).to(self.device)
+                (input_len,) = input.shape
+
+                # Since in _collate we make sure length is descending, the longest is always the first one.
+                padding_length = (
+                    padding_length if padding_length is not None else input_len
+                )
+
+                # Pad length from seq to padding_length
+                input = torch.cat(
+                    [
+                        input,  # [seq]
+                        torch.zeros(padding_length - input_len, dtype=torch.long).to(
+                            input.device
+                        ),  # [padding_length - seq]
+                    ],
+                    dim=0,
+                )
+                inputs.append(input.unsqueeze(0))  # [1, padding_length]
+                cont_tokens_list.append(continuation_enc)
+                input_lens.append(input_len)
+
+            batched_inputs = torch.cat(inputs, dim=0)  # [batch, padding_length]
+            multi_logits = F.log_softmax(
+                self._model_call(batched_inputs), dim=-1
+            ).cpu()  # [batch, padding_length, vocab]
+
+            for (cache_key, _, _), logits, input, input_len, cont_tokens in zip(
+                chunk, multi_logits, inputs, input_lens, cont_tokens_list
+            ):
+                # Slice to original seq length
+                cont_len = len(cont_tokens)
+                # [1, seq, vocab]
+                logits = logits[input_len - cont_len : input_len].unsqueeze(0)
+                # Check if per-token argmax is exactly equal to continuation
+                greedy_tokens = logits.argmax(dim=-1)
+                # [1, seq]
+                cont_tokens = torch.tensor(cont_tokens, dtype=torch.long).unsqueeze(0)
+                max_equal = (greedy_tokens == cont_tokens).all()
+
+                # Obtain logprobs at the corresponding continuation token indices
+                # last_token_slice = logits[:, -1, :].squeeze(0).tolist()
+                # [1, seq]
+                logits = torch.gather(logits, 2, cont_tokens.unsqueeze(-1)).squeeze(-1)
+                # Answer: (log prob, is-exact-match)
+                answer = (float(logits.sum()), bool(max_equal))
+                # Partial caching
+                if cache_key is not None:
+                    self.cache_hook.add_partial("loglikelihood", cache_key, answer)
+                results.append(answer)
+        return reorder.get_original(results)
+
+    @abc.abstractmethod
+    def _model_call(
+        self, inputs: TokenSequence, labels: Optional[TokenSequence] = None
+    ) -> TokenSequence:
+        """
+        Args:
+            inputs (TokenSequence):
+                A list of strings or torch tensor of shape [batch, sequence]
+                the size of sequence may vary from call to call.
+            labels (TokenSequence, optional, defaults to None):
+                A list of strings or torch tensor of shape [batch, sequence]
+                useful for sequence-to-sequence language models.
+
+        Returns:
+            A list of ints or torch tensor of shape [batch, sequence, vocab]
+            with the logits returned from the model.
+        """
+        pass
+
+    @abc.abstractmethod
+    def _model_generate(
+        self, inputs: TokenSequence, max_tokens: int, stop: Optional[List[str]] = None
+    ) -> Union[TokenSequence, List[str]]:
+        """
+        Args:
+            inputs (TokenSequence):
+                A list of strings/ints or torch tensor of shape [batch, sequence]
+                the size of sequence may vary from call to call.
+            max_tokens (int):
+                The maximum number of tokens to generate.
+            stop (List[str], optional, defaults to None):
+                A list of stopping sequences. If provided, the generation will
+                stop when any string sequence in the list is encountered.
+
+        Returns:
+            A list of ints/strings or a torch tensor of shape [batch, sequence, vocab]
+            with continuation tokens/string of the inputs.
+        """
+        pass
+
+
+def hash_args(attr, args):
+    data = json.dumps([attr] + list(args))
+    return hashlib.sha256(data.encode("utf-8")).hexdigest()
+
+
+class CachingLM:
+    def __init__(self, lm: LM, cache_db: str):
+        """LM wrapper that returns cached results if they exist, and uses the underlying LM if not.
+
+        Args:
+            lm (LM):
+                The underlying LM to use.
+            cache_db (str):
+                Path to the `cache` database.
+        """
+        from sqlitedict import SqliteDict
+
+        self.lm = lm
+        if os.path.dirname(cache_db):
+            os.makedirs(os.path.dirname(cache_db), exist_ok=True)
+        self.cache_db = cache_db
+        self.dbdict = SqliteDict(cache_db, autocommit=True)
+        # Add hook to lm
+        lm.set_cache_hook(self.get_cache_hook())
+
+    def __getattr__(self, attr):
+        def fn(requests):
+            res = []
+            remaining_reqs = []
+
+            # Figure out which ones are cached and which ones are new
+            for req in requests:
+                hsh = hash_args(attr, req)
+                if hsh in self.dbdict:
+                    ob = self.dbdict[hsh]
+
+                    assert ob is not None
+                    res.append(ob)
+                else:
+                    res.append(None)
+                    remaining_reqs.append(req)
+
+            # Actually run the LM on the requests that do not have cached results
+            rem_res = getattr(self.lm, attr)(remaining_reqs)
+
+            # Stick the new ones back into the list and also cache any of the new ones
+            resptr = 0
+            for req, r in zip(remaining_reqs, rem_res):
+                while res[resptr] is not None:
+                    resptr += 1
+
+                res[resptr] = r
+                # Caching
+                hsh = hash_args(attr, req)
+                self.dbdict[hsh] = r
+            self.dbdict.commit()
+            return res
+
+        return fn
+
+    def get_cache_hook(self):
+        return CacheHook(self)
+
+
+class CacheHook:
+    def __init__(self, cachinglm: CachingLM):
+        if cachinglm is None:
+            self.dbdict = None
+            return
+        self.dbdict = cachinglm.dbdict
+
+    def add_partial(self, attr, req, res):
+        if self.dbdict is None:
+            return
+        hsh = hash_args(attr, req)
+        self.dbdict[hsh] = res

evaluation-pipeline/lm_eval/api/request.py

@@ -0,0 +1,53 @@
+from typing import Any, Optional
+
+
+REQUEST_RETURN_LENGTHS = {
+    "loglikelihood": 2,
+    "greedy_until": None,
+    "loglikelihood_rolling": None,
+}
+
+
+class Request:
+    def __init__(
+        self, request_type: str, args: Optional[Any] = None, index: Optional[int] = None
+    ):
+        if request_type not in REQUEST_RETURN_LENGTHS.keys():
+            raise NotImplementedError(
+                "The request type {} is not implemented!".format(request_type)
+            )
+        self.request_type = request_type
+        self.args = args
+        self.index = index
+
+    def __iter__(self):
+        if REQUEST_RETURN_LENGTHS[self.request_type] is None:
+            raise IndexError("This request type does not return multiple arguments!")
+        for i in range(REQUEST_RETURN_LENGTHS[self.request_type]):
+            yield Request(self.request_type, self.args, i)
+
+    def __getitem__(self, i: int):
+        if REQUEST_RETURN_LENGTHS[self.request_type] is None:
+            raise IndexError("This request type does not return multiple arguments!")
+        return Request(self.request_type, self.args, i)
+
+    def __eq__(self, other: "Request"):
+        return (
+            self.request_type == other.request_type
+            and self.args == other.args
+            and self.index == other.index
+        )
+
+    def __repr__(self):
+        return f"Req_{self.request_type}{self.args}[{self.index}]\n"
+
+
+class RequestFactory:
+    def __getattr__(self, attr):
+        def fn(*args):
+            return Request(attr, args)
+
+        return fn
+
+
+rf = RequestFactory()

evaluation-pipeline/lm_eval/api/task.py

@@ -0,0 +1,874 @@
+import abc
+import logging
+import re
+import datasets
+import os
+import numpy as np
+import promptsource.templates
+from abc import abstractmethod
+from typing import Callable, List, Mapping, Optional, Tuple, Union
+
+from lm_eval.api import utils
+from lm_eval.api.metric import (
+    bits_per_byte,
+    bleu,
+    mean,
+    rouge,
+    sari,
+    weighted_perplexity,
+)
+from lm_eval.api.request import Request, rf
+
+
+logger = logging.getLogger(__name__)
+
+
+class Task(abc.ABC):
+    """A task represents an entire benchmark including its dataset, problems,
+    answers, and evaluation methods. See BoolQ for a simple example implementation
+
+    A `doc` can be any python object which represents one instance of evaluation.
+    This is usually a dictionary e.g.
+        {"question": ..., "answer": ...} or
+        {"question": ..., question, answer)
+    """
+
+    VERSION = 0
+
+    # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub
+    # or a path to a custom `datasets` loading script.
+    DATASET_PATH: str = None
+
+    # The name of a subset within `DATASET_PATH`.
+    DATASET_NAME: str = None
+
+    def __init__(
+        self,
+        data_dir: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+        download_mode: Optional[str] = None,
+        file_path: Optional[str] = None,
+    ):
+        """
+        Args:
+            data_dir (str, optional, defaults to None):
+                Stores the path to a local folder containing the `Task`'s data
+                files. Use this to specify the path to manually downloaded data
+                (usually when the dataset is not publicly accessible).
+            cache_dir (str, optional, defaults to None):
+                The directory to read/write the `Task` dataset. This follows the
+                HuggingFace `datasets` API with the default cache directory located
+                at:
+                    `~/.cache/huggingface/datasets`
+                NOTE: You can change the cache location globally for a given
+                process by setting the shell environment variable,
+                `HF_DATASETS_CACHE`, to another directory:
+                    `export HF_DATASETS_CACHE="/path/to/another/directory"`
+            download_mode (datasets.DownloadMode, optional, defaults to None):
+                How to treat pre-existing `Task` downloads and data.
+                - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS`
+                    Reuse download and reuse dataset.
+                - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS`
+                    Reuse download with fresh dataset.
+                - `datasets.DownloadMode.FORCE_REDOWNLOAD`
+                    Fresh download and fresh dataset.
+        """
+        if file_path:
+            self.load_from_file(file_path, cache_dir,
+                                download_mode=datasets.DownloadMode.FORCE_REDOWNLOAD)
+        else:
+            self.download(data_dir, cache_dir, download_mode)
+        self._training_docs = None
+        self._fewshot_docs = None
+
+    def download(
+        self,
+        data_dir: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+        download_mode: Optional[str] = None,
+    ):
+        """Downloads and returns the task dataset.
+
+        NOTE: Override this method to download the dataset from a custom API.
+        """
+        self.dataset = datasets.load_dataset(
+            path=self.DATASET_PATH,
+            name=self.DATASET_NAME,
+            data_dir=data_dir,
+            cache_dir=cache_dir,
+            download_mode=download_mode,
+        )
+
+    def load_from_file(
+        self,
+        file_path,
+        cache_dir: Optional[str] = None,
+        download_mode: Optional[str] = None,
+    ):
+        # get split names
+        splits = {}
+        dirname = os.path.dirname(file_path)
+        for filename in os.listdir(dirname):
+            if not filename.startswith(os.path.basename(file_path)):
+                continue
+            if filename.count(".") == 2:
+                splitname = filename.split(".")[1]
+                splits[splitname] = os.path.join(dirname, filename)
+            else:
+                splits["train"] = os.path.join(dirname, filename)
+
+        self.dataset = datasets.load_dataset(
+            "json",
+            data_files=splits,
+            cache_dir=cache_dir,
+            download_mode=download_mode,
+        )
+
+    @abstractmethod
+    def has_training_docs(self):
+        """Whether the task has a training set"""
+        pass
+
+    @abstractmethod
+    def has_validation_docs(self):
+        """Whether the task has a validation set"""
+        pass
+
+    @abstractmethod
+    def has_test_docs(self):
+        """Whether the task has a test set"""
+        pass
+
+    def training_docs(self) -> datasets.Dataset:
+        """
+        Returns:
+            A dataset of training documents.
+        """
+        return datasets.Dataset.from_dict({})
+
+    def validation_docs(self) -> datasets.Dataset:
+        """
+        Returns:
+            A dataset of validation documents.
+        """
+        return datasets.Dataset.from_dict({})
+
+    def test_docs(self) -> datasets.Dataset:
+        """
+        Returns:
+            A dataset of test documents.
+        """
+        return datasets.Dataset.from_dict({})
+
+    def _process_doc(self, doc):
+        """Override this to process (detokenize, strip, replace, etc.) individual
+        documents. This can be used in a map over documents of a data split.
+        E.g. `map(self._process_doc, self.dataset["validation"])`
+
+        Returns:
+            The processed version of the specified `doc`.
+        """
+        return doc
+
+    @abstractmethod
+    def doc_to_text(self, doc: dict) -> str:
+        pass
+
+    @abstractmethod
+    def doc_to_target(self, doc: dict) -> str:
+        pass
+
+    @abstractmethod
+    def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]:
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        Args:
+            doc (dict):
+                The document as returned from training_docs, validation_docs, or
+                test_docs.
+            ctx (str):
+                The context string, generated by fewshot_context. This includes
+                the natural language description, as well as the few shot examples,
+                and the question part of the document for `doc`.
+            args (dict):
+                The specifics of the context, including number of few shots.
+
+        Returns:
+            An iterable of `Request` objects.
+        """
+        pass
+
+    @abstractmethod
+    def process_results(
+        self, doc: dict, results: list
+    ) -> Union[dict, Tuple[dict, dict]]:
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of sub-metrics and values are the values of
+        the metric for that one document.
+
+        Args:
+            doc (dict):
+                The document as returned from training_docs, validation_docs, or
+                test_docs.
+            results (list):
+                The results of the requests created in construct_requests.
+
+        Returns:
+            A dict of metric results.
+        """
+        pass
+
+    @abstractmethod
+    def aggregation(self) -> Mapping[str, Callable]:
+        """
+        Returns:
+            A dictionary where keys are the names of sub-metrics and values are
+            functions that aggregate a list of metric scores.
+            {str: [metric_score] -> float}
+        """
+        pass
+
+    @abstractmethod
+    def higher_is_better(self) -> Mapping[str, bool]:
+        """
+        Returns:
+            A dictionary where keys are the names of sub-metrics and values are
+            whether a higher value of the sub-metric is better.
+            {str: bool}
+        """
+        pass
+
+
+class PromptSourceTask(Task):
+    """These are the metrics from promptsource that we have
+    added default behavior for. If you want to add default behavior for a new metric,
+    update the functions below. If you want to use one of the following metrics,
+    *and* add additional custom processing, override `process_results`, `higher_is_better`, and `aggregation`.
+    """
+
+    CONFIGURED_RANKED_CHOICE_PS_METRICS = {"Accuracy"}
+    CONFIGURED_GENERATION_PS_METRICS = {"BLEU", "ROUGE", "SARI"}
+    SPLIT = None
+
+    def __init__(
+        self,
+        data_dir: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+        download_mode: Optional[str] = None,
+        prompt_template: Optional[promptsource.templates.Template] = None,
+        example_separator: Optional[str] = "\n###\n",
+        text_target_separator: Optional[str] = " ",
+        save_examples: Optional[bool] = True,
+        file_path: Optional[str] = None,
+    ):
+        """
+        Args:
+            save_examples (bool, optional, defaults to True):
+                Whether to save each example and corresponding model predictions
+                to an output `dict`.
+
+            > Few-shot prompting args
+
+            example_separator (str, optional, defaults to '\n###\n'):
+                The string that will be used to separate the few-shot examples
+                from the prompt example.
+                Default: '\n###\n'
+                    See Webson & Pavlick (2022) https://arxiv.org/pdf/2109.01247.pdf
+                    for justification of this separator.
+            text_target_separator (str, optional, defaults to ' '):
+                The string that will be used to separate the prompt example
+                from the target text.
+                NOTE: This is assumed to be some form of whitespace-only separation,
+                    e.g. "\n\n", "\t", "  ", etc. Otherwise, you should update
+                    the Task's `promptsource` template with the appropriate
+                    separator(s).
+                Example:
+                    Q: Where is the Eiffel Tower located? A:{text_target_separator}Paris
+        """
+        assert isinstance(save_examples, bool), "`save_examples` must be a bool."
+        assert isinstance(example_separator, str) and isinstance(
+            text_target_separator, str
+        ), "Separator args must be strings."
+        assert (
+            text_target_separator.isspace()
+        ), f"`text_target_separator` must be whitespace only. Got: `{text_target_separator}`"
+
+        if file_path:
+            super().__init__(cache_dir=cache_dir, file_path=file_path,
+                             download_mode=download_mode)
+        else:
+            super().__init__(data_dir, cache_dir, download_mode)
+        self.prompt_template = prompt_template
+        self.save_examples = save_examples
+        self.example_separator = example_separator
+        self.text_target_separator = text_target_separator
+
+    def stop_sequences(self) -> List[str]:
+        """Denote where the generation should end based on the few-shot example
+        separator.
+
+        NOTE: Override this if you want to use a sequence other than just the
+        task's few-shot example separator.
+        """
+        return [self.example_separator]
+
+    def max_generation_length(self) -> Optional[int]:
+        """Denote where the max length of the generation if it is obvious from the task."""
+        return None
+
+    def evaluation_docs(self) -> datasets.Dataset:
+        """Returns the `dataset` split to be used for evaluation."""
+        if self.has_test_docs():
+            return self.test_docs()
+        elif self.has_validation_docs():
+            return self.validation_docs()
+        else:
+            raise RuntimeError("Task has neither test_docs nor validation_docs")
+
+    def fewshot_docs(self) -> datasets.Dataset:
+        """Returns the `dataset` split that the few-shot examples should be sample
+        from. This prioritizes the `train_docs` split as the few-shot example
+        source, then `validation_docs`, and lastly `test_docs`.
+        """
+        if self.has_training_docs():
+            return self.training_docs()
+        elif self.has_validation_docs():
+            return self.validation_docs()
+        else:
+            return self.test_docs()
+
+    def doc_to_text(self, doc: dict) -> str:
+        """Returns the input string for a particular example, given the hf dict."""
+        if self.prompt_template is None:
+            return self.null_prompt_doc_to_text(doc)
+        # is just a string
+        text, _ = self.prompt_template.apply(doc)
+        return text
+
+    def null_prompt_doc_to_text(self, doc: dict) -> str:
+        return NotImplementedError("Override this method in your task!")
+
+    def doc_to_target(self, doc: dict) -> List[str]:
+        """Returns the target string for a particular example, given the hf dict."""
+        if self.prompt_template is None:
+            return self.null_prompt_doc_to_target(doc)
+        # is a list of strings where it usually only has one element: the correct answer
+        _, target = self.prompt_template.apply(doc)
+        return target
+
+    def null_prompt_doc_to_target(self, doc: dict) -> List[str]:
+        return NotImplementedError("Override this method in your task!")
+
+    def doc_to_rawtext(self, doc: dict) -> str:
+        """This should be used for selecting the raw text of the document.
+
+        The current use case is for computing SARI which requires the text
+        without the prompt. The `text` field is not standardized across tasks
+        so this is task specific.
+        """
+        raise NotImplementedError("This is task specific.")
+
+    def invalid_doc_for_prompt(self, doc) -> bool:
+        """Some prompts may not work for some documents.
+        Default: False
+        """
+        return False
+
+    def format_example(self, text: str, target: str, separator: str) -> str:
+        """Returns the text and target combined by the specified `separator`"""
+        return text + separator + target
+
+    def null_prompt_answer_choices(self, doc: dict) -> List[str]:
+        return NotImplementedError("Override this method in your task!")
+
+    def fewshot_examples(
+        self,
+        docs: datasets.Dataset,
+        k: int,
+        rng: np.random.Generator,
+        prompt: dict = None,
+    ) -> Tuple[List[dict], List[int]]:
+        """Returns `k` random examples from the set of documents in `docs`.
+
+        Args:
+            docs (datasets.Dataset):
+                The dataset of documents to sample few-shot examples from.
+            k (int):
+                The number of few-shot examples.
+            rng (np.random.Generator):
+                The pseudo-random number generator used to randomly sample examples.
+            prompt (Optional[dict]):
+                The prompt document. Specify this to ensure the prompt is not in
+                the set of few-shot examples.
+
+        Returns:
+            A tuple of two lists. The first list contains the few-shot examples
+        """
+        random_indices = np.arange(len(docs)).tolist()
+        rng.shuffle(random_indices)
+
+        i = 0
+        fewshot_examples, fewshot_idx = [], []
+        for idx in random_indices:
+            if i >= k:  # Break when we have enough examples.
+                break
+            is_same_prompt = prompt is not None and all(
+                # Skips the `doc_id` key assigned to `prompt`s during eval pre-processing.
+                docs[idx][k] == prompt[k]
+                for k in docs[idx].keys()
+            )
+            if self.invalid_doc_for_prompt(docs[idx]) or is_same_prompt:
+                continue
+            fewshot_examples.append(docs[idx])
+            fewshot_idx.append(int(idx))
+            i += 1
+        return fewshot_examples, fewshot_idx
+
+    def fewshot_context(
+        self, doc: dict, num_fewshot: int, rng: Optional[np.random.Generator]
+    ) -> Tuple[str, dict]:
+        """Returns a few-shot context string made up of `num_fewshot` number of
+        labeled examples, and an appended prompt example without labeling.
+
+        Args:
+            doc (dict):
+                The document as returned from training_docs, validation_docs, or test_docs.
+            num_fewshot (int):
+                The number of fewshot examples to provide in the returned context string.
+            rng (numpy.random.Generator):
+                The pseudo-random number generator used to randomly sample few-shot examples.
+
+        Returns:
+            A few-shot context string and a dictionary containing few-shot context
+            logging information.
+                ctx (str):
+                    The fewshot context.
+                logging_info (dict):
+                    A `dict` of logging info that can be used to identify few-shot
+                    sources.
+        """
+        assert (
+            rng is not None
+        ), "A `numpy.random.Generator` argument must be provided to `rng`"
+
+        if num_fewshot == 0:
+            labeled_examples = ""
+            fewshot_idx, fewshot_target_idx, fewshot_src = ([], [], None)
+        else:
+            # Construct few-shot labeled examples.
+            fewshot_docs = self.fewshot_docs()
+            fewshot_src = str(fewshot_docs.split)
+            fewshot_examples, fewshot_idx = self.fewshot_examples(
+                fewshot_docs, k=num_fewshot, rng=rng, prompt=doc
+            )
+            labeled_examples_list = []
+            fewshot_target_idx = []
+            for fewshot_example in fewshot_examples:
+                text = self.doc_to_text(fewshot_example)
+                targets = self.doc_to_target(fewshot_example)
+                # Choose 1 random target from multi-reference targets.
+                target_idx = int(rng.integers(0, len(targets)))
+                target = targets[target_idx].strip()
+                labeled_examples_list.append(
+                    self.format_example(text, target, self.text_target_separator)
+                )
+                fewshot_target_idx.append(target_idx)
+            labeled_examples = self.example_separator.join(labeled_examples_list)
+            # Leave an extra `example_separator` right before the prompt.
+            labeled_examples += self.example_separator
+
+        prompt = self.doc_to_text(doc)
+        ctx = labeled_examples + prompt
+        logging_info = {
+            "fewshot_idx": fewshot_idx,
+            "fewshot_target_idx": fewshot_target_idx,
+            "fewshot_source": fewshot_src,
+            "fewshot_num": num_fewshot,
+            "ctx": ctx,
+        }
+        return ctx, logging_info
+
+    def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]:
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        Args:
+            doc (dict):
+                The document as returned from training_docs, validation_docs, or
+                test_docs.
+            ctx (str):
+                The context string, generated by fewshot_context. This includes
+                the natural language description, as well as the few shot examples,
+                and the question part of the document for `doc`.
+            args (dict):
+                The specifics of the context, including number of few shots.
+
+        Returns:
+            An iterable of `Request` objects.
+        """
+        requests = []
+        if self.prompt_template is None:
+            answer_choices_list = self.null_prompt_answer_choices(doc)
+        else:
+            answer_choices_list = self.prompt_template.get_answer_choices_list(doc)
+        if answer_choices_list:
+            # If answer_choices_list, then this is a ranked choice prompt.
+            for answer_choice in answer_choices_list:
+                ll_answer_choice, _ = rf.loglikelihood(
+                    ctx, self.text_target_separator + answer_choice
+                )
+                requests.append(ll_answer_choice)
+        else:
+            # If not, then this is a generation prompt.
+            request_args = {
+                "stop_sequences": self.stop_sequences(),
+                "max_generation_length": self.max_generation_length(),
+                "num_fewshot": args["num_fewshot"],
+            }
+            cont_request = rf.greedy_until(ctx, request_args)
+            requests.append(cont_request)
+        return requests
+
+    def process_results(
+        self, doc: dict, results: list
+    ) -> Union[dict, Tuple[dict, dict]]:
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of sub-metrics and values are the values of
+        the metric for that one document.
+
+        NOTE: This function automates processing by using the `promptsource`
+        metadata to determine the metric.
+
+        Args:
+            doc (dict):
+                The document as returned from training_docs, validation_docs, or
+                test_docs.
+            results (list):
+                The results of the requests created in construct_requests.
+
+        Returns:
+            A dict of metric results.
+        """
+        if self.prompt_template is None:
+            answer_choices_list = self.null_prompt_answer_choices(doc)
+        else:
+            answer_choices_list = self.prompt_template.get_answer_choices_list(doc)
+        target = self.doc_to_target(doc)
+        if answer_choices_list:
+            # If answer_choices_list, then this is a ranked choice prompt.
+            # NOTE: In the future, target could be a list of strings.
+            assert isinstance(target, list) and len(target) == 1
+            target = target[0].strip()
+            try:
+                target_idx = answer_choices_list.index(target)
+            except ValueError as e:
+                print("answer_choices_list:", answer_choices_list)
+                print("target:", target)
+                raise ValueError(e)
+
+            pred = answer_choices_list[np.argmax(results)]
+            out = {}
+            metric_list = ["Accuracy"]  # TODO: CLI framework for specifying metrics
+
+            if self.prompt_template:
+                metric_list = self.prompt_template.metadata.metrics
+            for metric in metric_list:
+                if metric not in self.CONFIGURED_RANKED_CHOICE_PS_METRICS:
+                    logger.warning(
+                        f"Unexpected metric: `{metric}`. Add it, or use a task-specific solution."
+                    )
+                if metric == "Accuracy":
+                    out["acc"] = pred == target
+                    # Byte-length normalization.
+                    completion_len = np.array(
+                        [float(len(i)) for i in answer_choices_list]
+                    )
+                    out["acc_norm"] = (
+                        1.0
+                        if np.argmax(results / completion_len) == target_idx
+                        else 0.0
+                    )
+            # TODO: Add metrics here.
+        else:
+            # If not, then this is a generation prompt.
+            # NOTE: In the future, target will be a list of strings.
+            assert isinstance(target, list)
+            pred = results[0].strip()
+            out = {}
+            for metric in self.prompt_template.metadata.metrics:
+                if metric not in self.CONFIGURED_GENERATION_PS_METRICS:
+                    logger.warning(
+                        f"Unexpected metric: `{metric}`. Add it, or use a task-specific solution."
+                    )
+                if metric == "BLEU":
+                    out["bleu"] = (target, pred)
+                elif metric == "ROUGE":
+                    # TODO: This computes all rouge sub-metrics. Find a generic
+                    # way to handle user specified rouge sub-metrics to avoid extra
+                    # compute.
+                    rouge_scores = rouge(target, pred)
+                    # Flatten rouge score dict.
+                    rouge_scores = utils.flatten(rouge_scores)
+                    # Merge all the rouge-type scores into the `out` dict.
+                    out = {**out, **rouge_scores}
+                elif metric == "SARI":
+                    out["sari"] = sari(self.doc_to_rawtext(doc), pred, target)
+
+        # TODO: Wrap process results s.t. override impl do not
+        # override the save examples.
+        if self.save_examples:
+            example = {
+                "pred": pred,
+                "target": target,
+                "answer_choices_list": answer_choices_list,
+            }
+            return out, example
+        return out
+
+    def aggregation(self) -> Mapping[str, Callable]:
+        out = {}
+        metric_list = ["Accuracy"]
+        if self.prompt_template:
+            metric_list = self.prompt_template.metadata.metrics
+        for metric in metric_list:
+            if metric == "Accuracy":
+                out["acc"] = mean
+                out["acc_norm"] = mean
+            elif metric == "BLEU":
+                out["bleu"] = bleu
+            elif metric == "ROUGE":
+                # TODO: Find a generic way to handle user specified rouge metrics.
+                out["rouge1_precision"] = mean
+                out["rouge1_recall"] = mean
+                out["rouge1_fmeasure"] = mean
+
+                out["rouge2_precision"] = mean
+                out["rouge2_recall"] = mean
+                out["rouge2_fmeasure"] = mean
+
+                out["rougeL_precision"] = mean
+                out["rougeL_recall"] = mean
+                out["rougeL_fmeasure"] = mean
+
+                out["rougeLsum_precision"] = mean
+                out["rougeLsum_recall"] = mean
+                out["rougeLsum_fmeasure"] = mean
+            elif metric == "SARI":
+                out["sari"] = mean
+        return out
+
+    def higher_is_better(self) -> Mapping[str, bool]:
+        out = {}
+        for metric in self.prompt_template.metadata.metrics:
+            if metric == "Accuracy":
+                out["acc"] = True
+                out["acc_norm"] = True
+            elif metric == "BLEU":
+                out["bleu"] = True
+            elif metric == "ROUGE":
+                # TODO: Find a generic way to handle user specified rouge metrics.
+                out["rouge1_precision"] = True
+                out["rouge1_recall"] = True
+                out["rouge1_fmeasure"] = True
+
+                out["rouge2_precision"] = True
+                out["rouge2_recall"] = True
+                out["rouge2_fmeasure"] = True
+
+                out["rougeL_precision"] = True
+                out["rougeL_recall"] = True
+                out["rougeL_fmeasure"] = True
+
+                out["rougeLsum_precision"] = True
+                out["rougeLsum_recall"] = True
+                out["rougeLsum_fmeasure"] = True
+            elif metric == "SARI":
+                out["sari"] = True
+        return out
+
+    def get_logging_info(self):
+        if self.prompt_template is None:
+            return self.null_prompt_get_logging_info()
+        return {
+            "fixed_answer_choice_list": self.prompt_template.get_fixed_answer_choices_list(),
+            "dataset_path": self.DATASET_PATH,
+            "dataset_name": self.DATASET_NAME,
+            "subset": self.SPLIT,
+            "prompt_name": self.prompt_template.get_name(),
+            "prompt_id": self.prompt_template.get_id(),
+            "prompt_jinja": self.prompt_template.jinja,
+            "prompt_original_task": self.prompt_template.metadata.original_task,
+            # Placeholder for comment in post-processing.
+            "comment": "",
+        }
+
+
+class TranslationTask(PromptSourceTask):
+
+    # Language specific functions.
+    @classmethod
+    def zh_split(cls, zh_text: str) -> List[str]:
+        """Chinese splitting"""
+        import jieba
+
+        return [" ".join(jieba.cut(txt.strip())) for txt in zh_text]
+
+    @classmethod
+    def ja_split(cls, ja_text: str) -> List[str]:
+        """Japanese splitting"""
+        import nagisa
+
+        return [" ".join(nagisa.tagging(txt.strip()).words) for txt in ja_text]
+
+    NO_SPACE_LANG = {"zh": zh_split, "ja": ja_split}
+
+    def invalid_doc_for_prompt(self, doc) -> bool:
+        # Skip docs with empty references.
+        if self.doc_to_target(doc) == [""]:
+            return True
+        return False
+
+    def _get_src_ref_codes(self, template_name: str) -> Tuple[str, str]:
+        """Returns a 2-tuple of (src_lang, ref_lang) codes from the prompt template name."""
+        # Get the lang codes from the dataset name.
+        lang_pairs = self.DATASET_NAME.split("-")
+        # Template name ordering defines the src and ref lang codes.
+        if self.DATASET_NAME in template_name:
+            return lang_pairs[0], lang_pairs[1]
+        # Flip the lang pairs following the prompt source.
+        return lang_pairs[1], lang_pairs[0]
+
+    def process_results(
+        self, doc: dict, results: list
+    ) -> Union[dict, Tuple[dict, dict]]:
+        answer_choices_list = self.prompt_template.get_answer_choices_list(doc)
+        target = self.doc_to_target(doc)
+
+        # Add spaces between words for BLEU score calculation of target languages like Chinese
+        _, tar_lang_code = self._get_src_ref_codes(self.prompt_template.name)
+        if tar_lang_code in self.NO_SPACE_LANG:
+            target = [self.NO_SPACE_LANG[tar_lang_code]([t])[0] for t in target]
+            results = self.NO_SPACE_LANG[tar_lang_code](results)
+        pred = results[0].strip()
+
+        out = {}
+        for metric in self.prompt_template.metadata.metrics:
+            assert (
+                metric in self.CONFIGURED_GENERATION_PS_METRICS
+            ), "Unexpected metric. Add it, or use a task-specific solution."
+            if metric == "BLEU":
+                out["bleu"] = (target, pred)
+            elif metric == "ROUGE":
+                # TODO: This computes all rouge sub-metrics. Find a generic
+                # way to handle user specified rouge sub-metrics to avoid extra
+                # compute.
+                rouge_scores = rouge(target, pred)
+                # Flatten rouge score dict.
+                rouge_scores = utils.flatten(rouge_scores)
+                # Merge all the rouge-type scores into the `out` dict.
+                out = {**out, **rouge_scores}
+
+        # TODO: Wrap process results s.t. override impl do not
+        # override the save examples.
+        if self.save_examples:
+            example = {
+                "pred": pred,
+                "target": target,
+                "answer_choices_list": answer_choices_list,
+            }
+            return out, example
+        return out
+
+
+class PerplexityTask(PromptSourceTask):
+    """NOTE: Prompts are ignored for perplexity tasks."""
+
+    def doc_to_text(self, doc: dict) -> str:
+        return ""
+
+    def doc_to_target(self, doc: dict) -> List[str]:
+        """Because prompts are ignored, return the relevant text from doc."""
+        raise NotImplementedError()
+
+    def fewshot_context(
+        self,
+        doc: dict,
+        num_fewshot: int,
+        rng: Optional[np.random.Generator],
+    ) -> Tuple[str, dict]:
+        assert (
+            num_fewshot == 0
+        ), "The number of fewshot examples must be 0 for perplexity tasks."
+        assert (
+            rng is not None
+        ), "A `numpy.random.Generator` argument must be provided to `rng`"
+        return (
+            "",
+            {
+                "fewshot_idx": [],
+                "fewshot_target_idx": [],
+                "fewshot_source": None,
+                "fewshot_num": 0,
+                "ctx": "",
+            },
+        )
+
+    def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]:
+        assert not ctx
+        string = self.doc_to_target(doc)[0]
+        req = rf.loglikelihood_rolling(string)
+        return req
+
+    def process_results(
+        self, doc: dict, results: list
+    ) -> Union[dict, Tuple[dict, dict]]:
+        (loglikelihood,) = results
+        target = self.doc_to_target(doc)[0]
+        words = self.count_words(target)
+        bytes_ = self.count_bytes(target)
+
+        out = {
+            "word_perplexity": (loglikelihood, words),
+            "byte_perplexity": (loglikelihood, bytes_),
+            "bits_per_byte": (loglikelihood, bytes_),
+        }
+        if self.save_examples:
+            return out, {
+                "word_perplexity_instance": weighted_perplexity(
+                    [(loglikelihood, words)]
+                ),
+                "byte_perplexity_instance": weighted_perplexity(
+                    [(loglikelihood, bytes_)]
+                ),
+                "bits_per_byte_instance": bits_per_byte([(loglikelihood, bytes_)]),
+            }
+        return out
+
+    def aggregation(self) -> Mapping[str, Callable]:
+        return {
+            "word_perplexity": weighted_perplexity,
+            "byte_perplexity": weighted_perplexity,
+            "bits_per_byte": bits_per_byte,
+        }
+
+    def higher_is_better(self) -> Mapping[str, bool]:
+        return {
+            "word_perplexity": False,
+            "byte_perplexity": False,
+            "bits_per_byte": False,
+        }
+
+    @classmethod
+    def count_bytes(cls, doc):
+        return len(doc.encode("utf-8"))
+
+    @classmethod
+    def count_words(cls, doc):
+        """Downstream tasks with custom word boundaries should override this!"""
+        return len(re.split(r"\s+", doc))
+
+    def get_logging_info(self):
+        return {
+            "prompt_name": None,
+        }

evaluation-pipeline/lm_eval/api/utils.py

@@ -0,0 +1,357 @@
+import collections
+import pathlib
+import re
+import sys
+import torch
+from typing import Callable, Final, Iterable, List, Optional, Tuple, Union
+from collections.abc import MutableMapping
+from transformers import set_seed as transformers_set_seed
+
+
+# General Utils
+
+
+class ExitCodeError(Exception):
+    pass
+
+
+# Reproducibility utils
+
+
+DEFAULT_SEED: Final[int] = 1234
+
+
+def set_seed(seed: Optional[int] = DEFAULT_SEED):
+    transformers_set_seed(seed)
+
+
+# Token Utils
+
+
+def general_detokenize(s: str) -> str:
+    s = s.replace(" n't", "n't")
+    s = s.replace(" )", ")")
+    s = s.replace("( ", "(")
+    s = s.replace('" ', '"')
+    s = s.replace(' "', '"')
+    s = re.sub(r" (['.,])", r"\1", s)
+    return s
+
+
+def get_rolling_token_windows(
+    token_list: List[int], prefix_token: int, max_seq_len: int, context_len: int
+) -> Iterable[Tuple[List[int], List[int]]]:
+    """Returns a generator of rolling windows of length `max_seq_len` from a list of tokens.
+
+    Args:
+        token_list (List[int]):
+            List of tokens to be predicted.
+        prefix_token (int):
+            Dummy token like <eos> so the first token has something to condition
+            on.
+        max_seq_len (int):
+            The maximum sequence length of the model or a length we want to use.
+        context_len (int):
+            Amount of desired token context for prediction. Needs to be at least 1.
+            This allows for a rolling window context, letting each prediction
+            window to potentially condition on some context.
+
+    Returns:
+        Generator of tuples: (input_tokens, pred_tokens)
+        NOTE: Score only the last len(pred_tokens) logits of the LM.
+    """
+    assert 1 <= context_len <= max_seq_len
+    if not token_list:
+        return
+    # +1 offset, going from input->preds
+    pred_len = max_seq_len - context_len + 1
+    predicted = 0
+
+    # Special handling for first window: predict all tokens
+    first_seq_len = min(max_seq_len, len(token_list))
+    yield [prefix_token] + token_list[: first_seq_len - 1], token_list[:first_seq_len]
+    predicted += first_seq_len
+
+    while predicted < len(token_list):
+        window_pred_len = min(len(token_list) - predicted, pred_len)
+        window_end = predicted + window_pred_len
+
+        yield (
+            token_list[window_end - max_seq_len - 1 : window_end - 1],
+            token_list[window_end - window_pred_len : window_end],
+        )
+        predicted += window_pred_len
+
+
+def split_and_pad_windows(
+    windows: List[Tuple[str, str]], pad_token_id: int, max_seq_len: int
+) -> Tuple[List[int], List[int]]:
+    """Splits and pads a sequence of rolling context and continuation windows
+    from `get_rolling_token_windows`.
+
+    Example:
+        [
+            ([1] , [23, 19, 3]),  # (context, continuation)
+            ([43], [2, 4]])
+        ]
+
+    Output:
+        [
+            [[1],[43]],                # Split & padded contexts.
+            [[23, 19, 3], [2, 4, 1]]`  # Split & padded continuations.
+        ]
+        where `1` = `pad_token` id.
+
+    Args:
+        windows (List[Tuple[str, str]]):
+            A generator of rolling `(context, continuation)` token windows
+            (tuples).
+        pad_token_id (int):
+            The token id to pad with.
+        max_seq_len (int):
+            The maximum sequence length of the model or a length we want to use.
+
+    Returns:
+        A tuple of (context, continuation) padding windows.
+    """
+    contexts, continuations = zip(*windows)
+    contexts, continuations = list(contexts), list(continuations)
+
+    # Pad contexts:
+    rollover_context = contexts[-1]
+    rollover_context_size = len(rollover_context)
+    # Handle empty final context token list - just add 1 token.
+    if rollover_context_size == 0:
+        contexts[-1] += [pad_token_id]
+    elif rollover_context_size > 1:
+        for i in range(len(contexts[:-1])):
+            contexts[i] += [pad_token_id] * (rollover_context_size - len(contexts[i]))
+
+    # Pad continuations:
+    rollover_continuation = continuations[-1]
+    rollover_continuation_size = len(rollover_continuation)
+    is_multiple_windows = len(continuations) > 1
+    if rollover_continuation_size < max_seq_len and is_multiple_windows:
+        continuations[-1] = rollover_continuation + [pad_token_id] * (
+            max_seq_len - rollover_continuation_size
+        )
+    return contexts, continuations
+
+
+def make_disjoint_window(pair):
+    """Takes output from get_rolling_token_windows and makes the context not
+    overlap with the continuation.
+    """
+    a, b = pair
+    return a[: -(len(b) - 1)], b
+
+
+def select_continuation_from_batch_left_padding(
+    generations: Union[List[List[int]], torch.Tensor], max_context_size: int
+):
+    """Select the continuation from the batch, removing prompts of different lengths.
+
+    Args:
+        generations (Union[List[List[int]], torch.Tensor]):
+            A tensor or list-of-lists of shape [batch_size, sequence length].
+        max_context_size (int):
+            The size of the biggest context; generations will proceed from that
+            index.
+
+    Example:
+        PAD     PAD Continue : The dog chased the cat  [every       day of the week]
+        Riddle  me    this   : The  dog chased the  cat [yesterday] PAD PAD PAD PAD
+
+    Output:
+        [every day of the week]
+        [yesterday]  PAD PAD PAD PAD
+    """
+    return generations[:, max_context_size:]
+
+
+# Container Utils
+
+
+class Reorderer:
+    def __init__(self, arr, fn):
+        self.size = len(arr)
+        arr = list(enumerate(arr))
+        arr = group(arr, lambda x: fn(x[1]))
+        arr = [([y[0] for y in x], x[0][1]) for x in arr]
+        arr.sort(key=lambda x: fn(x[1]))
+        self.arr = arr
+
+    def get_reordered(self):
+        return [x[1] for x in self.arr]
+
+    def get_original(self, newarr):
+        res = [None] * self.size
+        cov = [False] * self.size
+        for (inds, _), v in zip(self.arr, newarr):
+            for ind in inds:
+                res[ind] = v
+                cov[ind] = True
+        assert all(cov)
+        return res
+
+
+def flatten(
+    d: Union[dict, MutableMapping],
+    parent_key: str = "",
+    sep: str = "_",
+) -> dict:
+    # From: https://stackoverflow.com/a/6027615
+    items = []
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, MutableMapping):
+            items.extend(flatten(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def join_iters(iterables: Iterable) -> List:
+    for iterable in iterables:
+        yield from iterable
+
+
+def chunks(iterable: Iterable, n: int) -> List:
+    arr = []
+    for x in iterable:
+        arr.append(x)
+        if len(arr) == n:
+            yield arr
+            arr = []
+    if arr:
+        yield arr
+
+
+def group(arr: Iterable, fn: Callable) -> List:
+    res = collections.defaultdict(list)
+    for ob in arr:
+        res[fn(ob)].append(ob)
+    return list(res.values())
+
+
+# CLI utils
+
+
+def cli_template_names(
+    task_name: str, template_names: str, template_idx: int = None
+) -> List[str]:
+    """Returns a selection of template names for a given task and comma-
+    separated string of template names.
+
+    Example:
+        cli_template_names("task", "A,B,C") -> ["A", "B", "C"]
+
+    Args:
+        task_name (str):
+            Name of the task from which to retrieve template names.
+        template_names (str):
+            A string of template names separated by a comma if multiple names
+            are given.
+            General Selectors:
+                "all_templates":
+                    Returns all templates for the task.
+                "original_templates":
+                    Returns all templates with formatting that matches the
+                    original task design.
+        template_idx (int, optional, defaults to None):
+            If given, returns only the template at the given index.
+
+    Returns:
+        A list of template names.
+    """
+    import lm_eval.tasks
+
+    if template_names == "all_templates":
+        selections = lm_eval.tasks.list_templates(task_name)
+    elif template_names == "original_templates":
+        templates = lm_eval.tasks.get_templates(task_name)
+        selections = []
+        for name in templates.all_template_names:
+            if templates[name].metadata.original_task is True:
+                selections.append(name)
+        if not selections:
+            raise ValueError(f"No original task templates found for {task_name}")
+    else:
+        selections = template_names.split(",")
+    if template_idx is not None:
+        selections = [selections[template_idx]]
+    return selections
+
+
+def parse_cli_args_string(args: str) -> dict:
+    """Parses a string in the following format to a kwargs dictionary.
+    "args1=val1,arg2=val2"
+    """
+    # Remove leading whitespace but not trailing in case a `val` contains necessary whitespace.
+    args = args.lstrip()
+    if not args:
+        return {}
+    arg_list = args.split(",")
+    args_dict = {}
+    for arg in arg_list:
+        # Split on the first `=` to allow for `=`s in `val`.
+        k, v = arg.split("=", 1)
+        args_dict[k] = str_to_builtin_type(v)
+    return args_dict
+
+
+def str_to_builtin_type(s: str) -> str:
+    for fn in (to_bool, int, float):
+        try:
+            return fn(s)
+        except ValueError:
+            pass
+    return s
+
+
+# https://stackoverflow.com/questions/7019283/automatically-type-cast-parameters-in-python
+def to_bool(s: str):
+    if s == "True" or s == "true":
+        return True
+    if s == "False" or s == "false":
+        return False
+    raise ValueError(f"The input `{s}` is not of boolean form.")
+
+
+# Test utils
+
+
+def find_test_root(*, start_path: pathlib.Path) -> pathlib.Path:
+    """Search upward in the directory tree to a maximum of three layers
+    to find and return the package root (containing the 'tests' folder)
+    """
+    cur_path = start_path.resolve()
+    max_layers = 3
+    for _ in range(max_layers):
+        if (cur_path / "tests" / "test_version_stable.py").exists():
+            return cur_path
+        else:
+            cur_path = cur_path.parent.resolve()
+    raise FileNotFoundError(
+        f"Unable to find package root within {max_layers} upwards" + f"of {start_path}"
+    )
+
+
+def run_task_tests(*, task_list: List[str]):
+    """Find the package root and run the tests for the given tasks."""
+    import pytest
+
+    package_root = find_test_root(start_path=pathlib.Path(__file__))
+    task_string = " or ".join(task_list)
+    args = [
+        f"{package_root}/tests/test_version_stable.py",
+        f"--rootdir={package_root}",
+        "-k",
+        f"{task_string}",
+    ]
+    sys.path.append(str(package_root))
+    pytest_return_val = pytest.main(args)
+    if pytest_return_val:
+        raise ValueError(
+            f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}"
+        )

evaluation-pipeline/lm_eval/datasets/README.md

@@ -0,0 +1,6 @@
+# datasets
+
+This directory contains custom EleutherAI datasets not available in the HuggingFace `datasets` hub.
+
+In the rare case that you need to add a custom dataset to this collection, follow the
+HuggingFace `datasets` guide found [here](https://huggingface.co/docs/datasets/dataset_script).

evaluation-pipeline/lm_eval/datasets/arithmetic/arithmetic.py

@@ -0,0 +1,216 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""GPT-3 Arithmetic Test Dataset."""
+
+
+import json
+
+import datasets
+
+
+_CITATION = """\
+@inproceedings{NEURIPS2020_1457c0d6,
+    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
+    booktitle = {Advances in Neural Information Processing Systems},
+    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
+    pages = {1877--1901},
+    publisher = {Curran Associates, Inc.},
+    title = {Language Models are Few-Shot Learners},
+    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},
+    volume = {33},
+    year = {2020}
+}
+"""
+
+_DESCRIPTION = """\
+A small battery of 10 tests that involve asking language models a simple arithmetic
+problem in natural language.
+"""
+
+_HOMEPAGE = "https://github.com/openai/gpt-3/tree/master/data"
+
+# TODO: Add the licence for the dataset here if you can find it
+_LICENSE = ""
+
+
+class ArithmeticConfig(datasets.BuilderConfig):
+    """BuilderConfig for GPT3 Arithmetic Test Dataset."""
+
+    def __init__(self, url, features, **kwargs):
+        """BuilderConfig for GPT3 Arithmetic dataset.
+
+        Args:
+        url: *string*, the url to the specific subset of the GPT3 Arithmetic dataset.
+        features: *list[string]*, list of the features that will appear in the
+            feature dict.
+        """
+        # Version history:
+        super().__init__(version=datasets.Version("0.0.1"), **kwargs)
+        self.url = url
+        self.features = features
+
+
+class Arithmetic(datasets.GeneratorBasedBuilder):
+    """A small battery of 10 tests involving simple arithmetic problems."""
+
+    BUILDER_CONFIGS = [
+        ArithmeticConfig(
+            name="arithmetic_2da",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_addition.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="2-digit addition",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_2ds",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_subtraction.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="2-digit subtraction",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_3da",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_addition.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="3-digit addition",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_3ds",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_subtraction.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="3-digit subtraction",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_4da",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_addition.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="4-digit addition",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_4ds",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_subtraction.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="4-digit subtraction",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_5da",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_addition.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="5-digit addition",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_5ds",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_subtraction.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="5-digit subtraction",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_2dm",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_multiplication.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="2-digit multiplication",
+        ),
+        ArithmeticConfig(
+            name="arithmetic_1dc",
+            url="https://raw.githubusercontent.com/openai/gpt-3/master/data/single_digit_three_ops.jsonl",
+            features=datasets.Features(
+                {
+                    "context": datasets.Value("string"),
+                    "completion": datasets.Value("string"),
+                }
+            ),
+            description="Single digit 3 operations",
+        ),
+    ]
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=f"{_DESCRIPTION}\n{self.config.description}",
+            features=self.config.features,
+            homepage=_HOMEPAGE,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        urls = self.config.url
+        data_dir = dl_manager.download_and_extract(urls)
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={
+                    "filepath": data_dir,
+                    "split": datasets.Split.VALIDATION,
+                },
+            ),
+        ]
+
+    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+    def _generate_examples(self, filepath, split):
+        with open(filepath, encoding="utf-8") as f:
+            for key, row in enumerate(f):
+                data = json.loads(row)
+                context = (
+                    data["context"]
+                    .strip()
+                    .replace("\n\n", "\n")
+                    .replace("Q:", "Question:")
+                    .replace("A:", "Answer:")
+                )
+                completion = data["completion"]
+                yield key, {"context": context, "completion": completion}

evaluation-pipeline/lm_eval/datasets/arithmetic/dataset_infos.json

@@ -0,0 +1 @@
+{"arithmetic_2da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 96624, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_addition.jsonl": {"num_bytes": 138624, "checksum": "75a54b7a3db3b23369df74fe440c23025f3d3c51f664300bd3d56632b2617b3d"}}, "download_size": 138624, "post_processing_size": null, "dataset_size": 96624, "size_in_bytes": 235248}, "arithmetic_2ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 98216, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_subtraction.jsonl": {"num_bytes": 140216, "checksum": "da956066ff108c00b341d360567472784f5fd872d6465071b44a14291205bc03"}}, "download_size": 140216, "post_processing_size": null, "dataset_size": 98216, "size_in_bytes": 238432}, "arithmetic_3da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n3-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_3da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 102612, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_addition.jsonl": {"num_bytes": 144612, "checksum": "124865e30efd2abfbc1855dd34c218fc02d32d780ace970ab9b4ea3fa74c798b"}}, "download_size": 144612, "post_processing_size": null, "dataset_size": 102612, "size_in_bytes": 247224}, "arithmetic_3ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n3-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_3ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 104150, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_subtraction.jsonl": {"num_bytes": 146150, "checksum": "7fc6aaedcb0e2bd17c398dd4147c5585b1e608278a8e98b914e69656707d6a29"}}, "download_size": 146150, "post_processing_size": null, "dataset_size": 104150, "size_in_bytes": 250300}, "arithmetic_4da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n4-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_4da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 108570, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_addition.jsonl": {"num_bytes": 150570, "checksum": "459c6f75baa2e8d7cf50bdd07db6d0ca9133a6b137d95d09267db85b6e07f391"}}, "download_size": 150570, "post_processing_size": null, "dataset_size": 108570, "size_in_bytes": 259140}, "arithmetic_4ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n4-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_4ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 110150, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_subtraction.jsonl": {"num_bytes": 152150, "checksum": "0c47db40a10c052ef0cf732a9ef2edaa53d66377d43eb47a9c382d33a8af7102"}}, "download_size": 152150, "post_processing_size": null, "dataset_size": 110150, "size_in_bytes": 262300}, "arithmetic_5da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n5-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_5da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 114476, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_addition.jsonl": {"num_bytes": 156476, "checksum": "30ada42efe315b958c6e9649274005d3b720e50298e92c3a2d321f8996e58f54"}}, "download_size": 156476, "post_processing_size": null, "dataset_size": 114476, "size_in_bytes": 270952}, "arithmetic_5ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n5-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_5ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 116119, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_subtraction.jsonl": {"num_bytes": 158119, "checksum": "8b98ccfc943cbf9193bcf1984954aa0b1a4527016072d972a2b055cc1482ca3c"}}, "download_size": 158119, "post_processing_size": null, "dataset_size": 116119, "size_in_bytes": 274238}, "arithmetic_2dm": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit multiplication", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2dm", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 100685, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_multiplication.jsonl": {"num_bytes": 142685, "checksum": "5613d1d1cc3b2c03edc1990252247d34c10ec82944b2cdeb19e71b00f237f431"}}, "download_size": 142685, "post_processing_size": null, "dataset_size": 100685, "size_in_bytes": 243370}, "arithmetic_1dc": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\nSingle digit 3 operations", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n    author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n    booktitle = {Advances in Neural Information Processing Systems},\n    editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n    pages = {1877--1901},\n    publisher = {Curran Associates, Inc.},\n    title = {Language Models are Few-Shot Learners},\n    url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n    volume = {33},\n    year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_1dc", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 97651, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/single_digit_three_ops.jsonl": {"num_bytes": 139651, "checksum": "08b34e3272a8ff1d4932d63f251519d14c485c38d582366e1e323d0b859c3925"}}, "download_size": 139651, "post_processing_size": null, "dataset_size": 97651, "size_in_bytes": 237302}}

evaluation-pipeline/lm_eval/datasets/asdiv/asdiv.py

@@ -0,0 +1,111 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ASDIV dataset."""
+
+
+import os
+import xml.etree.ElementTree as ET
+
+import datasets
+
+
+_CITATION = """\
+@misc{miao2021diverse,
+    title={A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers},
+    author={Shen-Yun Miao and Chao-Chun Liang and Keh-Yih Su},
+    year={2021},
+    eprint={2106.15772},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI}
+}
+"""
+
+_DESCRIPTION = """\
+ASDiv (Academia Sinica Diverse MWP Dataset) is a diverse (in terms of both language
+patterns and problem types) English math word problem (MWP) corpus for evaluating
+the capability of various MWP solvers. Existing MWP corpora for studying AI progress
+remain limited either in language usage patterns or in problem types. We thus present
+a new English MWP corpus with 2,305 MWPs that cover more text patterns and most problem
+types taught in elementary school. Each MWP is annotated with its problem type and grade
+level (for indicating the level of difficulty).
+"""
+
+_HOMEPAGE = "https://github.com/chaochun/nlu-asdiv-dataset"
+
+# TODO: Add the licence for the dataset here if you can find it
+_LICENSE = ""
+
+_URLS = "https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip"
+
+
+class ASDiv(datasets.GeneratorBasedBuilder):
+    """ASDiv: A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers"""
+
+    VERSION = datasets.Version("0.0.1")
+
+    BUILDER_CONFIGS = [
+        datasets.BuilderConfig(
+            name="asdiv",
+            version=VERSION,
+            description="A diverse corpus for evaluating and developing english math word problem solvers",
+        )
+    ]
+
+    def _info(self):
+        features = datasets.Features(
+            {
+                "body": datasets.Value("string"),
+                "question": datasets.Value("string"),
+                "solution_type": datasets.Value("string"),
+                "answer": datasets.Value("string"),
+                "formula": datasets.Value("string"),
+            }
+        )
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=features,
+            homepage=_HOMEPAGE,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        urls = _URLS
+        data_dir = dl_manager.download_and_extract(urls)
+        base_filepath = "nlu-asdiv-dataset-55790e5270bb91ccfa5053194b25732534696b50"
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={
+                    "filepath": os.path.join(
+                        data_dir, base_filepath, "dataset", "ASDiv.xml"
+                    ),
+                    "split": datasets.Split.VALIDATION,
+                },
+            ),
+        ]
+
+    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+    def _generate_examples(self, filepath, split):
+        tree = ET.parse(filepath)
+        root = tree.getroot()
+        for key, problem in enumerate(root.iter("Problem")):
+            yield key, {
+                "body": problem.find("Body").text,
+                "question": problem.find("Question").text,
+                "solution_type": problem.find("Solution-Type").text,
+                "answer": problem.find("Answer").text,
+                "formula": problem.find("Formula").text,
+            }

evaluation-pipeline/lm_eval/datasets/asdiv/dataset_infos.json

@@ -0,0 +1 @@
+{"asdiv": {"description": "ASDiv (Academia Sinica Diverse MWP Dataset) is a diverse (in terms of both language\npatterns and problem types) English math word problem (MWP) corpus for evaluating\nthe capability of various MWP solvers. Existing MWP corpora for studying AI progress\nremain limited either in language usage patterns or in problem types. We thus present\na new English MWP corpus with 2,305 MWPs that cover more text patterns and most problem\ntypes taught in elementary school. Each MWP is annotated with its problem type and grade\nlevel (for indicating the level of difficulty).\n", "citation": "@misc{miao2021diverse,\n    title={A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers},\n    author={Shen-Yun Miao and Chao-Chun Liang and Keh-Yih Su},\n    year={2021},\n    eprint={2106.15772},\n    archivePrefix={arXiv},\n    primaryClass={cs.AI}\n}\n", "homepage": "https://github.com/chaochun/nlu-asdiv-dataset", "license": "", "features": {"body": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "solution_type": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"dtype": "string", "id": null, "_type": "Value"}, "formula": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "as_div", "config_name": "asdiv", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 501489, "num_examples": 2305, "dataset_name": "as_div"}}, "download_checksums": {"https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip": {"num_bytes": 440966, "checksum": "8f1fe4f6d5f170ec1e24ab78c244153c14c568b1bb2b1dad0324e71f37939a2d"}}, "download_size": 440966, "post_processing_size": null, "dataset_size": 501489, "size_in_bytes": 942455}}

evaluation-pipeline/lm_eval/datasets/coqa/coqa.py

@@ -0,0 +1,245 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CoQA dataset.
+
+This `CoQA` adds the "additional_answers" feature that's missing in the original
+datasets version:
+https://github.com/huggingface/datasets/blob/master/datasets/coqa/coqa.py
+"""
+
+
+import json
+
+import datasets
+
+
+_CITATION = """\
+@misc{reddy2018coqa,
+    title={CoQA: A Conversational Question Answering Challenge},
+    author={Siva Reddy and Danqi Chen and Christopher D. Manning},
+    year={2018},
+    eprint={1808.07042},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+"""
+
+_DESCRIPTION = """\
+CoQA is a large-scale dataset for building Conversational Question Answering
+systems. The goal of the CoQA challenge is to measure the ability of machines to
+understand a text passage and answer a series of interconnected questions that
+appear in a conversation.
+"""
+
+_HOMEPAGE = "https://stanfordnlp.github.io/coqa/"
+
+# TODO: Add the licence for the dataset here if you can find it
+_LICENSE = ""
+
+_URLS = {
+    "train": "https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json",
+    "validation": "https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json",
+}
+
+# `additional_answers` are not available in the train set so we fill them with
+# empty dicts of the same form.
+_EMPTY_ADDITIONAL_ANSWER = {
+    "0": [
+        {
+            "span_start": -1,
+            "span_end": -1,
+            "span_text": "",
+            "input_text": "",
+            "turn_id": -1,
+        }
+    ],
+    "1": [
+        {
+            "span_start": -1,
+            "span_end": -1,
+            "span_text": "",
+            "input_text": "",
+            "turn_id": -1,
+        }
+    ],
+    "2": [
+        {
+            "span_start": -1,
+            "span_end": -1,
+            "span_text": "",
+            "input_text": "",
+            "turn_id": -1,
+        }
+    ],
+}
+
+
+class Coqa(datasets.GeneratorBasedBuilder):
+    """CoQA is a large-scale dataset for building Conversational Question Answering systems."""
+
+    VERSION = datasets.Version("0.0.1")
+
+    BUILDER_CONFIGS = [
+        datasets.BuilderConfig(
+            name="coqa", version=VERSION, description="The CoQA dataset."
+        ),
+    ]
+
+    def _info(self):
+        features = datasets.Features(
+            {
+                "id": datasets.Value("string"),
+                "source": datasets.Value("string"),
+                "story": datasets.Value("string"),
+                "questions": datasets.features.Sequence(
+                    {
+                        "input_text": datasets.Value("string"),
+                        "turn_id": datasets.Value("int32"),
+                    }
+                ),
+                "answers": datasets.features.Sequence(
+                    {
+                        "span_start": datasets.Value("int32"),
+                        "span_end": datasets.Value("int32"),
+                        "span_text": datasets.Value("string"),
+                        "input_text": datasets.Value("string"),
+                        "turn_id": datasets.Value("int32"),
+                    }
+                ),
+                "additional_answers": {
+                    "0": datasets.features.Sequence(
+                        {
+                            "span_start": datasets.Value("int32"),
+                            "span_end": datasets.Value("int32"),
+                            "span_text": datasets.Value("string"),
+                            "input_text": datasets.Value("string"),
+                            "turn_id": datasets.Value("int32"),
+                        }
+                    ),
+                    "1": datasets.features.Sequence(
+                        {
+                            "span_start": datasets.Value("int32"),
+                            "span_end": datasets.Value("int32"),
+                            "span_text": datasets.Value("string"),
+                            "input_text": datasets.Value("string"),
+                            "turn_id": datasets.Value("int32"),
+                        }
+                    ),
+                    "2": datasets.features.Sequence(
+                        {
+                            "span_start": datasets.Value("int32"),
+                            "span_end": datasets.Value("int32"),
+                            "span_text": datasets.Value("string"),
+                            "input_text": datasets.Value("string"),
+                            "turn_id": datasets.Value("int32"),
+                        }
+                    ),
+                },
+            }
+        )
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=features,
+            homepage=_HOMEPAGE,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        urls = {"train": _URLS["train"], "validation": _URLS["validation"]}
+        data_dirs = dl_manager.download_and_extract(urls)
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={
+                    "filepath": data_dirs["train"],
+                    "split": datasets.Split.TRAIN,
+                },
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={
+                    "filepath": data_dirs["validation"],
+                    "split": datasets.Split.VALIDATION,
+                },
+            ),
+        ]
+
+    # method parameters are unpacked from `gen_kwargs` as given in `_split_generators`
+    def _generate_examples(self, filepath, split):
+        with open(filepath, encoding="utf-8") as f:
+            data = json.load(f)
+            for row in data["data"]:
+                id = row["id"]
+                source = row["source"]
+                story = row["story"]
+                questions = [
+                    {"input_text": q["input_text"], "turn_id": q["turn_id"]}
+                    for q in row["questions"]
+                ]
+                answers = [
+                    {
+                        "span_start": a["span_start"],
+                        "span_end": a["span_end"],
+                        "span_text": a["span_text"],
+                        "input_text": a["input_text"],
+                        "turn_id": a["turn_id"],
+                    }
+                    for a in row["answers"]
+                ]
+                if split == datasets.Split.TRAIN:
+                    additional_answers = _EMPTY_ADDITIONAL_ANSWER
+                else:
+                    additional_answers = {
+                        "0": [
+                            {
+                                "span_start": a0["span_start"],
+                                "span_end": a0["span_end"],
+                                "span_text": a0["span_text"],
+                                "input_text": a0["input_text"],
+                                "turn_id": a0["turn_id"],
+                            }
+                            for a0 in row["additional_answers"]["0"]
+                        ],
+                        "1": [
+                            {
+                                "span_start": a1["span_start"],
+                                "span_end": a1["span_end"],
+                                "span_text": a1["span_text"],
+                                "input_text": a1["input_text"],
+                                "turn_id": a1["turn_id"],
+                            }
+                            for a1 in row["additional_answers"]["1"]
+                        ],
+                        "2": [
+                            {
+                                "span_start": a2["span_start"],
+                                "span_end": a2["span_end"],
+                                "span_text": a2["span_text"],
+                                "input_text": a2["input_text"],
+                                "turn_id": a2["turn_id"],
+                            }
+                            for a2 in row["additional_answers"]["2"]
+                        ],
+                    }
+                yield row["id"], {
+                    "id": id,
+                    "story": story,
+                    "source": source,
+                    "questions": questions,
+                    "answers": answers,
+                    "additional_answers": additional_answers,
+                }

evaluation-pipeline/lm_eval/datasets/coqa/dataset_infos.json

@@ -0,0 +1 @@
+{"coqa": {"description": "CoQA is a large-scale dataset for building Conversational Question Answering\nsystems. The goal of the CoQA challenge is to measure the ability of machines to\nunderstand a text passage and answer a series of interconnected questions that\nappear in a conversation.\n", "citation": "@misc{reddy2018coqa,\n    title={CoQA: A Conversational Question Answering Challenge},\n    author={Siva Reddy and Danqi Chen and Christopher D. Manning},\n    year={2018},\n    eprint={1808.07042},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL}\n}\n", "homepage": "https://stanfordnlp.github.io/coqa/", "license": "", "features": {"id": {"dtype": "string", "id": null, "_type": "Value"}, "source": {"dtype": "string", "id": null, "_type": "Value"}, "story": {"dtype": "string", "id": null, "_type": "Value"}, "questions": {"feature": {"input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "answers": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "additional_answers": {"0": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "1": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "2": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "coqa", "config_name": "coqa", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 26250528, "num_examples": 7199, "dataset_name": "coqa"}, "validation": {"name": "validation", "num_bytes": 3765933, "num_examples": 500, "dataset_name": "coqa"}}, "download_checksums": {"https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json": {"num_bytes": 49001836, "checksum": "b0fdb2bc1bd38dd3ca2ce5fa2ac3e02c6288ac914f241ac409a655ffb6619fa6"}, "https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json": {"num_bytes": 9090845, "checksum": "dfa367a9733ce53222918d0231d9b3bedc2b8ee831a2845f62dfc70701f2540a"}}, "download_size": 58092681, "post_processing_size": null, "dataset_size": 30016461, "size_in_bytes": 88109142}}

evaluation-pipeline/lm_eval/datasets/drop/dataset_infos.json

@@ -0,0 +1 @@
+{"drop": {"description": "DROP is a QA dataset which tests comprehensive understanding of paragraphs. In \nthis crowdsourced, adversarially-created, 96k question-answering benchmark, a \nsystem must resolve multiple references in a question, map them onto a paragraph,\nand perform discrete operations over them (such as addition, counting, or sorting).\n", "citation": "@misc{dua2019drop,\n    title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, \n    author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},\n    year={2019},\n    eprint={1903.00161},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL}\n}\n", "homepage": "https://allenai.org/data/drop", "license": "", "features": {"section_id": {"dtype": "string", "id": null, "_type": "Value"}, "passage": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "query_id": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"number": {"dtype": "string", "id": null, "_type": "Value"}, "date": {"day": {"dtype": "string", "id": null, "_type": "Value"}, "month": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}}, "spans": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "worker_id": {"dtype": "string", "id": null, "_type": "Value"}, "hit_id": {"dtype": "string", "id": null, "_type": "Value"}}, "validated_answers": {"feature": {"number": {"dtype": "string", "id": null, "_type": "Value"}, "date": {"day": {"dtype": "string", "id": null, "_type": "Value"}, "month": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}}, "spans": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "worker_id": {"dtype": "string", "id": null, "_type": "Value"}, "hit_id": {"dtype": "string", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "drop", "config_name": "drop", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 108858121, "num_examples": 77409, "dataset_name": "drop"}, "validation": {"name": "validation", "num_bytes": 12560739, "num_examples": 9536, "dataset_name": "drop"}}, "download_checksums": {"https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip": {"num_bytes": 8308692, "checksum": "39d2278a29fd729de301b111a45f434c24834f40df8f4ff116d864589e3249d6"}}, "download_size": 8308692, "post_processing_size": null, "dataset_size": 121418860, "size_in_bytes": 129727552}}