Publish final TinyStories five-arm evaluation

ts-fivearm/evaluation/genppl-entropy-matched-gpt2-large-512-five-arm-v2.json

@@ -0,0 +1,639 @@
+{
+  "protocol": {
+    "reference_model": "gpt2-large",
+    "official_hf_metric": "evaluate.load('perplexity', module_type='metric')",
+    "hf_add_start_token": true,
+    "hf_aggregation": "arithmetic mean of per-example perplexities",
+    "entropy": "mean per-story unigram entropy in NCM BPE-8192 space, nats",
+    "n_samples": 512,
+    "seed": 42,
+    "n_provided_levels": 0,
+    "level0_condition": "learned BOS query",
+    "top_k": 50,
+    "top_p": 0.95,
+    "temperatures": [
+      0.6,
+      0.75,
+      0.9,
+      1.0,
+      1.1,
+      1.25,
+      1.5
+    ]
+  },
+  "metadata": {
+    "two_stage": {
+      "checkpoint": "outputs/ncm/tinystories-two-stage-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "two_stage"
+    },
+    "naive_e2e": {
+      "checkpoint": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "naive_e2e"
+    },
+    "eostok_apr": {
+      "checkpoint": "outputs/ncm/tinystories-eostok-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "eostok_apr"
+    },
+    "detached_apr": {
+      "checkpoint": "outputs/ncm/tinystories-detached-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "detached_apr"
+    },
+    "mstok_v1": {
+      "checkpoint": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "mstok_v1"
+    }
+  },
+  "curves": {
+    "two_stage": [
+      {
+        "temperature": 0.6,
+        "mean_entropy_nats": 3.5681163728432184,
+        "std_entropy_nats": 0.12139439092589657,
+        "hf_mean_perplexity": 34.04097021184862,
+        "median_example_perplexity": 30.19317054748535
+      },
+      {
+        "temperature": 0.75,
+        "mean_entropy_nats": 3.5792986346060314,
+        "std_entropy_nats": 0.11737114301086707,
+        "hf_mean_perplexity": 37.4731461815536,
+        "median_example_perplexity": 32.82475280761719
+      },
+      {
+        "temperature": 0.9,
+        "mean_entropy_nats": 3.6094152037721825,
+        "std_entropy_nats": 0.11177244278406301,
+        "hf_mean_perplexity": 43.146870439872146,
+        "median_example_perplexity": 37.35891342163086
+      },
+      {
+        "temperature": 1.0,
+        "mean_entropy_nats": 3.616894227996327,
+        "std_entropy_nats": 0.10971631990597369,
+        "hf_mean_perplexity": 46.59727036766708,
+        "median_example_perplexity": 40.54710006713867
+      },
+      {
+        "temperature": 1.1,
+        "mean_entropy_nats": 3.62595637200416,
+        "std_entropy_nats": 0.10419502403658659,
+        "hf_mean_perplexity": 51.24228238128126,
+        "median_example_perplexity": 43.895856857299805
+      },
+      {
+        "temperature": 1.25,
+        "mean_entropy_nats": 3.6428456713707456,
+        "std_entropy_nats": 0.10488014281690947,
+        "hf_mean_perplexity": 59.2652478441596,
+        "median_example_perplexity": 50.991899490356445
+      },
+      {
+        "temperature": 1.5,
+        "mean_entropy_nats": 3.6673122682942103,
+        "std_entropy_nats": 0.10621471090902414,
+        "hf_mean_perplexity": 78.60297981277108,
+        "median_example_perplexity": 67.421875
+      }
+    ],
+    "naive_e2e": [
+      {
+        "temperature": 0.6,
+        "mean_entropy_nats": 3.5478684929605695,
+        "std_entropy_nats": 0.13239739050981292,
+        "hf_mean_perplexity": 68.07085863873363,
+        "median_example_perplexity": 37.70510292053223
+      },
+      {
+        "temperature": 0.75,
+        "mean_entropy_nats": 3.5781703810566876,
+        "std_entropy_nats": 0.13058574202766063,
+        "hf_mean_perplexity": 75.42180791683495,
+        "median_example_perplexity": 40.19425392150879
+      },
+      {
+        "temperature": 0.9,
+        "mean_entropy_nats": 3.606539367449947,
+        "std_entropy_nats": 0.11804517358031621,
+        "hf_mean_perplexity": 84.82909641601145,
+        "median_example_perplexity": 46.33236885070801
+      },
+      {
+        "temperature": 1.0,
+        "mean_entropy_nats": 3.625497092519505,
+        "std_entropy_nats": 0.11221369856530092,
+        "hf_mean_perplexity": 94.89091111533344,
+        "median_example_perplexity": 51.998064041137695
+      },
+      {
+        "temperature": 1.1,
+        "mean_entropy_nats": 3.6367996859376883,
+        "std_entropy_nats": 0.11550294730010713,
+        "hf_mean_perplexity": 100.87178237363696,
+        "median_example_perplexity": 54.62735557556152
+      },
+      {
+        "temperature": 1.25,
+        "mean_entropy_nats": 3.651695341345158,
+        "std_entropy_nats": 0.11540239317885399,
+        "hf_mean_perplexity": 117.3333791512996,
+        "median_example_perplexity": 60.59077262878418
+      },
+      {
+        "temperature": 1.5,
+        "mean_entropy_nats": 3.6760182846680514,
+        "std_entropy_nats": 0.11288432736867066,
+        "hf_mean_perplexity": 147.5805096719414,
+        "median_example_perplexity": 85.40291213989258
+      }
+    ],
+    "eostok_apr": [
+      {
+        "temperature": 0.6,
+        "mean_entropy_nats": 3.5732832960192793,
+        "std_entropy_nats": 0.11066370820092461,
+        "hf_mean_perplexity": 27.021129613742232,
+        "median_example_perplexity": 24.01330852508545
+      },
+      {
+        "temperature": 0.75,
+        "mean_entropy_nats": 3.5872540310943526,
+        "std_entropy_nats": 0.11342775711458045,
+        "hf_mean_perplexity": 29.97356158774346,
+        "median_example_perplexity": 27.17264175415039
+      },
+      {
+        "temperature": 0.9,
+        "mean_entropy_nats": 3.6069873289458343,
+        "std_entropy_nats": 0.10656324933118969,
+        "hf_mean_perplexity": 33.702150508761406,
+        "median_example_perplexity": 30.7090425491333
+      },
+      {
+        "temperature": 1.0,
+        "mean_entropy_nats": 3.6115428540258163,
+        "std_entropy_nats": 0.10961490650565317,
+        "hf_mean_perplexity": 37.74269070290029,
+        "median_example_perplexity": 33.8486213684082
+      },
+      {
+        "temperature": 1.1,
+        "mean_entropy_nats": 3.6251551481246893,
+        "std_entropy_nats": 0.10399614350116333,
+        "hf_mean_perplexity": 41.20823755860329,
+        "median_example_perplexity": 36.7203311920166
+      },
+      {
+        "temperature": 1.25,
+        "mean_entropy_nats": 3.640281723888813,
+        "std_entropy_nats": 0.10838603320456089,
+        "hf_mean_perplexity": 48.322812678292394,
+        "median_example_perplexity": 41.99285888671875
+      },
+      {
+        "temperature": 1.5,
+        "mean_entropy_nats": 3.661071770697972,
+        "std_entropy_nats": 0.10534005508908131,
+        "hf_mean_perplexity": 64.73706209100783,
+        "median_example_perplexity": 55.88966369628906
+      }
+    ],
+    "detached_apr": [
+      {
+        "temperature": 0.6,
+        "mean_entropy_nats": 3.549042068352458,
+        "std_entropy_nats": 0.12114786549798884,
+        "hf_mean_perplexity": 32.145011492073536,
+        "median_example_perplexity": 27.757308959960938
+      },
+      {
+        "temperature": 0.75,
+        "mean_entropy_nats": 3.5746174490121083,
+        "std_entropy_nats": 0.11932749522209155,
+        "hf_mean_perplexity": 35.360570304095745,
+        "median_example_perplexity": 31.598602294921875
+      },
+      {
+        "temperature": 0.9,
+        "mean_entropy_nats": 3.589599591450561,
+        "std_entropy_nats": 0.11139944003880774,
+        "hf_mean_perplexity": 42.18865744769573,
+        "median_example_perplexity": 37.05057907104492
+      },
+      {
+        "temperature": 1.0,
+        "mean_entropy_nats": 3.60176389184409,
+        "std_entropy_nats": 0.1093612930954545,
+        "hf_mean_perplexity": 45.35750689730048,
+        "median_example_perplexity": 39.72639846801758
+      },
+      {
+        "temperature": 1.1,
+        "mean_entropy_nats": 3.6126366877843745,
+        "std_entropy_nats": 0.11016725214923163,
+        "hf_mean_perplexity": 49.00315220095217,
+        "median_example_perplexity": 43.20043182373047
+      },
+      {
+        "temperature": 1.25,
+        "mean_entropy_nats": 3.6269427581672815,
+        "std_entropy_nats": 0.11213443097599127,
+        "hf_mean_perplexity": 59.065537167713046,
+        "median_example_perplexity": 51.44508171081543
+      },
+      {
+        "temperature": 1.5,
+        "mean_entropy_nats": 3.654677036914653,
+        "std_entropy_nats": 0.1105355714349414,
+        "hf_mean_perplexity": 77.31420361995697,
+        "median_example_perplexity": 65.02359008789062
+      }
+    ],
+    "mstok_v1": [
+      {
+        "temperature": 0.6,
+        "mean_entropy_nats": 3.5682390111921922,
+        "std_entropy_nats": 0.11122234179384705,
+        "hf_mean_perplexity": 31.209586836397648,
+        "median_example_perplexity": 28.298173904418945
+      },
+      {
+        "temperature": 0.75,
+        "mean_entropy_nats": 3.582745350220796,
+        "std_entropy_nats": 0.11267712660982124,
+        "hf_mean_perplexity": 35.59679423645139,
+        "median_example_perplexity": 31.502530097961426
+      },
+      {
+        "temperature": 0.9,
+        "mean_entropy_nats": 3.6070369198933254,
+        "std_entropy_nats": 0.11233575823247058,
+        "hf_mean_perplexity": 41.070367699489,
+        "median_example_perplexity": 36.600229263305664
+      },
+      {
+        "temperature": 1.0,
+        "mean_entropy_nats": 3.618831043867462,
+        "std_entropy_nats": 0.10752506009027538,
+        "hf_mean_perplexity": 44.25286133028567,
+        "median_example_perplexity": 39.68633270263672
+      },
+      {
+        "temperature": 1.1,
+        "mean_entropy_nats": 3.631441237697225,
+        "std_entropy_nats": 0.10759170643925861,
+        "hf_mean_perplexity": 50.46371474303305,
+        "median_example_perplexity": 44.67747688293457
+      },
+      {
+        "temperature": 1.25,
+        "mean_entropy_nats": 3.6417516917586124,
+        "std_entropy_nats": 0.09997622821957552,
+        "hf_mean_perplexity": 58.558864971622825,
+        "median_example_perplexity": 52.33473587036133
+      },
+      {
+        "temperature": 1.5,
+        "mean_entropy_nats": 3.663654647597124,
+        "std_entropy_nats": 0.0983044595336084,
+        "hf_mean_perplexity": 76.8341717235744,
+        "median_example_perplexity": 66.00213623046875
+      }
+    ]
+  },
+  "matched_entropy": {
+    "range": [
+      3.5732832960192793,
+      3.654677036914653
+    ],
+    "targets": [
+      {
+        "target_entropy_nats": 3.5732832960192793,
+        "hf_mean_ppl": {
+          "two_stage": 35.62685596003978,
+          "naive_e2e": 74.23624766020004,
+          "eostok_apr": 27.021129613742232,
+          "detached_apr": 35.19282901141401,
+          "mstok_v1": 32.735149121805314
+        }
+      },
+      {
+        "target_entropy_nats": 3.5936317312431227,
+        "hf_mean_ppl": {
+          "two_stage": 40.17338861747122,
+          "naive_e2e": 80.5488639129262,
+          "eostok_apr": 31.17862230771295,
+          "detached_apr": 43.23904617117589,
+          "mstok_v1": 38.049801887896386
+        }
+      },
+      {
+        "target_entropy_nats": 3.613980166466966,
+        "hf_mean_ppl": {
+          "two_stage": 45.25288674365941,
+          "naive_e2e": 88.77830132585397,
+          "eostok_apr": 38.36320480621312,
+          "detached_apr": 49.94810768747251,
+          "mstok_v1": 42.94391405580331
+        }
+      },
+      {
+        "target_entropy_nats": 3.6343286016908096,
+        "hf_mean_ppl": {
+          "two_stage": 55.21936326340282,
+          "naive_e2e": 99.56418541759436,
+          "eostok_apr": 45.52284421650495,
+          "detached_apr": 63.92529199159157,
+          "mstok_v1": 52.73069980048283
+        }
+      },
+      {
+        "target_entropy_nats": 3.654677036914653,
+        "hf_mean_ppl": {
+          "two_stage": 68.6164369950842,
+          "naive_e2e": 121.04130766414731,
+          "eostok_apr": 59.68826368228699,
+          "detached_apr": 77.31420361995697,
+          "mstok_v1": 69.34346634429059
+        }
+      }
+    ],
+    "interpolation": "piecewise linear in measured mean entropy"
+  },
+  "samples": {
+    "two_stage": {
+      "temperature_0.6": [
+        " a picture, again. This time, there was a big picture, big, colorful picture the recipe.\nAmy and her mom printed the picture together. They learned that coloring it is important to be good but it it's also important to be responsible and take care of things. The moral of the story is that we",
+        " ground. She picked it up and put it on the ground. The little fly was very happy.\nThe next day, the fly. The small fly was very happy and said, \"Thank you, you you, Lily! You saved my fly.\" Lily was happy to help. And they lived happily ever after.",
+        ", a little girl named Lily came to play with her toy and her toy cat. She said, \"Oh, no! Boo is dirty. We need to bathe, and I will help you.\"\nLily and the toy snake played together all day. They laughed, jumped and played together. They played all day,",
+        " a very happy dog.\nOne day, a curious little and lawyer went to the park. They saw a big tree and wanted to mark it up. \"Be careful, bird,\" the boy said, \"This is a nice tree. Sharing means that something good is.\" The boy learned that it is important to be"
+      ],
+      "temperature_0.75": [
+        " but there was no bug inside. In,, there was a big mess and took the bug away.\nAmy and her friends cleaned up the hospital. They learned that it's important to care and, but it's also important to be careful and take take care of things. The moral of the story is that we",
+        " on the ground lay on the ground and trembled on the ground. She was was very very happy.\nThe next day, the saw how small Lily was very happy and said \"thank you and\" and \"Thank you, Lily! You are kind.\" Lily was happy to help. And they lived happily ever after.",
+        ", a little girl named Lily came to play with her toy car. \"Oh, no!\" Sally said. \"Oh, you're dirty! You need to rest, and I will help you.\"\nSo Lucy and the toy truck played all day. They spun, laughed, and laughed. They played all day,",
+        " were very happy.\nOne sunny day, the boy and a girl went to the park. They saw a big tree and wanted to climb it up. \"Be careful, boy,\" the boy said. \"This is tree. It is important to listen to music.\" But the boy learned that it is important to listen"
+      ],
+      "temperature_0.9": [
+        " there was no treasure there. In the park, there was a big, shiny rock.\nSue took the stone and showed her family all their dinner. They all learned it is important to for others, and it is important to be humble and be grateful for care others too. The moral of the story is that we",
+        " bug on the ground. He picked it with a soft leaf. The little bug was very happy.\nThe next day, the little bug was very very small and said, \"thank you\" said to Bob. \"Thank you, Bob! You are the best bug ever!\" And they lived happily happily ever after.",
+        ", a little girl named Lily came to play with her toy car. \"Oh, no!\" she said. \"Oh, you're dirty, you will hide, and I will see you soon.\"\nLily and the toy truck played together all day. They ran and jumped together. They laughed and played all day,",
+        " is a happy town.\nOne sunny day, a boy and squirrel went to the park. It saw a big tree and Penny wanted to climb it. \"Be careful, bird,\" said the boy. \"This tree is useful. It is important to it. Remember listen to others Wisdom and listen to and help others"
+      ],
+      "temperature_1": [
+        " there was no treasure there. In the park, there was a big, shiny.\nSue took the rock home, and shared it with her dad. They were all so happy that that it is important to help others. Tim learned that helping others can make him happier too. The moral of the story is that when",
+        " letter on the ground. She noticed it lying on the ground. She picked it up. The peanut was very happy. The little package belonged to a little boy named Tim.\nTim smiled and said \"thank you you, Lily! You found my peanut.\"\nThen, something unexpected happened.\nAs the peanut,",
+        " friend, a cat her friend came to play. The toy touched her toy. \"Oh no!\" She said. \"Ouch! This will hurt, hurt you, and I will get you soon.\"\nThe cat and the toy played together all day. They laughed, talked, and laughed. They played all day,",
+        " it was happy.\nOne sunny day, a boy and a squirrel went to the park. It saw a big tree and wanted warn him to be careful. \"Be,\" said the squirrel, \"but the tree is nice, but it is important.\" The boy learned to listen to the others and help others when he"
+      ],
+      "temperature_1.1": [
+        " there no one was there. In the park, there was a big, shiny.\nSue took the rock home, and shared it with her dad. They were all so happy that it's important to help others. Tim learned that that helping others can make him happier too. The moral of the story is that when",
+        " letter on the ground. She noticed it lying on the ground. She picked it up. The peanut was very happy. The little envelope belonged from a little boy named Tim.\nTim smiled and said \"thank you you, Lily! You found my peanut.\"\nThen, something unexpected happened.\nAs the peanut,",
+        " friend, the friend, friend came to play. The toy touched her toy. \"Oh no!\" She said. \"Ouch! This will hurt you. I will help you get rid of it.\"\nThe fairy and the toy played together all day. They laughed, talked, and laughed. They played all day,",
+        " it was happy.\nOne sunny day, a small squirrel and squirrel went to the park. It saw a big tree and wanted to warn to be careful. \"Be,\" said the squirrel, \"but the tree is nice, but it is important..\" They learned to listen to others and to help others when they"
+      ],
+      "temperature_1.25": [
+        " there was no one there. In the park, there was a big, shiny.\nSue took the stone home, and shared it with her dad. They were all so happy that it's important to help others. Tim also learned that helping others can make him happier too. The moral of the story is that when",
+        " envelope on the ground. It noticed it lying on the ground. She picked it up. The folder was very happy. The wet envelope belonged from a little boy named Tim. The mailman smiled and said \"thank you you, Lily! You found my letter.\"\nThen, something unexpected happened. The envelope rain came down",
+        " friend, a bird wanted her friend to play. The toy snatched her toy. \"Oh no!\" She said. \"Ouch! This will hurt you. I will help you get out and play.\"\nThe fairy and the bird played together all day. They laughed, talked, and laughed. They played all day,",
+        " she was happy.\nOne sunny day, a small, intelligent squirrel went to the park. It saw a well and wanted to play near it. Lily said \"Be careful, squirrel, said, \"Remember tree is interesting, but it is important.\" The squirrel learned to listen to the others and help others.."
+      ],
+      "temperature_1.5": [
+        " there there was one one until leaving the park, there was a big, shiny.\nSue took the stone home, and shared it with her dad. They were all so happy that it was grateful to help others. Tim learned that helping others made them right. The moral of the story is when happy that made made",
+        " One day. One day, he met a gentle talking cat. The cat was very big and mild, kind. One day, he baked the reward and gave it to cat. Tom was happy he said \"Thank you. They were, my friend.\"\nAfter they played ball, something unexpected happened. The cat spoke",
+        " friend, a bird bird friend tried to play. The toy snatched her toy. \"Oh no!\" She said. \"Ouch! This is hurt you. I will help you get out and play.\"\nThe sweet girl and the bird played all day. They laughed, ate, and laughed. They played all day,",
+        " boy was happy.\nOne sunny day, the boy saw a tree thick, old tree. He wanted to do climb it A bird came him. \"Be careful bird Bob the squirrel, said \"An the tree is interesting, but it is important to listen.\" The bird nodded to trust the and listened to others."
+      ]
+    },
+    "naive_e2e": {
+      "temperature_0.6": [
+        " too!\"  \nThe boy and girl smiled and said, \"It's so much fun!\" \nThe pilot smiled, waved back his arm and waved in his arms. He was sure he wasn the bear wouldn't fall in the truck, making sure it was safe safe. \n \n \nThe",
+        "\nThe little boy was so excited that he was amazed. \nHe then asked, \u201cCan you fly?\u201d \nThe star smiled, \u201cYes, please!\u201d \n \n \nHe carefully placed itself in the sky. \nThe star admired the little girl\u2019s.eve and heights.",
+        " they arrived, the couple arrived at the stadium.\nWhen he arrived at the stadium, he saw was so excited to see it. He was so excited to begin it! \n\n \nHe scararled around and waved around the other peck. \nThe two friends shared the games instruments together and",
+        "In the house, there was a little girl named Sue. Sue was very kind. Tom said, \"Don't worry, I will help find your way.\"\nJust then, something unexpected happened. A big, of strong people animals came out of the town! They became best friends, and Tom and Sue became Sue"
+      ],
+      "temperature_0.75": [
+        " man, \" \"Would you like the hat?\"\nThe child smiled and said yes. The man nodded again and smiled. \nThe girl smiled. She realized she didn't understand why exactly. He asked the man, \"How did you feel too much too?\"\nThe man smiled and said, \"It was",
+        " The boy was so so proud that he was so happy! \nHe then asked, \u201cDid you speak?\u201d \nThe unicorn winked, \u201cYes, please!\u201d \n \nThe elephant was happy and he inviting beauty the knowledge. \nThe boy watched the little frog\u2019s beautiful beautiful pecss",
+        " they arrived at the airportground and arrived.\nWhen they arrived at the airport he saw the aeroplane was so excited to see it. He was so excited to see and lots of wonderful surprises. \nThe End end.  The airport were full of joy. \nThe friends said goodbye and said goodbye to each",
+        " In the castle, there was a little girl named Sue. Sue was kind. Tom said, \"Don't worry, I will always be to be your friend.\"\nJust then, something unexpected happened. A big, little bird flew and landed on her shoulder too. They became best friends, and Tim and Sue became"
+      ],
+      "temperature_0.9": [
+        "Would you like a hedgehog?\"\nThe girl smiled and and said, \"meow!\" The barber nodded and nodded smiled. \n girl girl smiled. She realized she didn't understand why't. Then the girl asked the man \"Why did you feel too much?\"\nThe man smiled and said, \"It was",
+        " said that the person probably said it was so upset. \nShe was so upset that she just couldn't believe that she died. She didn't think could forget the mean witch. \nThe nightmare was gone and it slowly away slowly. \nThe little girl's heart were reappfes again. She",
+        " he arrived at the guests arrived at school.\nWhen the neighbours arrived, he saw the oat he was so excited to try it. He was so excited! There were lots of tasty seaweed. \n\n \nThe children had visited lots of delicious trophies. \nThe family started and started eating and eating",
+        " In the castle, there was a little girl named Sue. Sue was very kind. Tom said, \"Don't worry, Sue just wants to be your friend.\"\nJust then, something unexpected happened. A bird flew into the castle and landed on's pet too. They became good friends and from from that day on"
+      ],
+      "temperature_1": [
+        " girl replied, \"Would you like a joke?\"\nThe rabbit smiled and said yes. The old man nodded and smiled. \nThe girl smiled. She realized she didn't understand why still. Then the girl asked the man \"Why did you complain too?\"\nThe old man smiled and said, \"It was",
+        ", and the voice said that it was so upset. \nShe was so ignorant that she just couldn't believe that she died. She didn't think she forget the mean witch. \nThe nightmare was gone and he slowly disappeared away. \nThe little girl's heart were reappfes again. She",
+        " they arrived at the bakery arrived at school weekend.\nWhen he arrived, he saw the oat he was so excited to try it. He was so excited! He got some of sweets chocolates. \n\n \nThe children loved both kinds of delicious delicious waffles. \nThe family started licking and started eating their",
+        " In this town, there lived a small girl named Sue. Sue was scared of the storm. Tom said, \"Don't worry, I will find your home.\"\nJust then, something unexpected happened. A bird flew into a town and landed on Sue bed too. They became became best friends. From that day on"
+      ],
+      "temperature_1.1": [
+        ", \"Do you like the plane?\"\nThe girl nodded and nodded. \u201cHe looks so cool!\"\nThe two friends laughed. The two plane and Jack stayed hands in the sky still cold, but the plane didn't fall and and opening it was too hot. \nThey kept playing in the plane airplane until",
+        ", and the voice said that it was very upset. \nThe girl was so excited that she didn't notice the warning that her mom didn't mind Even though damaged caused her. \nThe nightmare was gone and alone on the ground. \nThe little girl's tears got of pantantssant",
+        "The two of them went into the picnic benches.\nWhen he arrived, he saw the oat he was so excited to try it. He was so excited! He tried some of the sweets. \n\n \nThe children started both kinds of hundred radishes. \nThe children started laughing and started eating all the",
+        " In this town, there lived a small girl named Sue. Sue was scared of the storm. Tom said, \"Don't worry, I will find your home.\"\nJust then, something unexpected happened. A bird flew into a town and landed on Sue bed too. They became became best friends. From that day on"
+      ],
+      "temperature_1.25": [
+        " replied, \"Do you like the plane?\"\nThe rabbit nodded eagerly. \" looks so interested!\". \nThe two bears laughed and hugged goodbye. Jack stayed with his in the boat still cold, but the rabbit realised it hadn't let them was too late! \nThey kept on sailing in the airplane, and",
+        ", and the voice said that it was so upset. \nThe girl was so excited that she didn't notice the warning that her mom didn't mind Even though damaged caused her. \nThe nightmare was gone and alone on the ground. \nThe little girl's tearsace of pantantsantant",
+        "The two of them went off into the picnic.\nWhen he arrived, he saw the oat and George so excited to try it. He was so excited! ran after some of it surprises.\n\n\n \nThe children cheered in half of food radishes. \nThe family happily gathered the rice and started eating",
+        " In the town, there lived a small girl named Sue. Sue was scared of the storm. Tom said, \"Don't worry, we will find your home.\"\nJust then, something unexpected happened. A bird flew into a window and landed on Sue bed too. They became best friends. And from that day on"
+      ],
+      "temperature_1.5": [
+        " day, a little named Lisa and her's brother came in the freezer with their freezer. Mia urged her mom, \"Can I do this chores?\" Her mom said yes, and she opened the complete. the dishwasher and said, \"Oh, there there's very noisy bird inside!\nLisa and her mom finished the",
+        ", and that the voice said it was so upset. \nThe girl was scared she couldn't hear about her but but was worried she didn't mind. She ignored her despiteiness. \nThe punishment was gone alone on the ground. \nThe little girl stepped closer onto her pambers and examined her",
+        "The two of them went off the school room.\nWhen they reached the classroom, he saw the area of the people! He was so excited! He was to eat some of more surprises.  \n \nHer children cheered bothturedies of waffles. \nThe family happily welcomed spoons and started eating their",
+        "In this town, there lived a little girl named Sue. Sue was scared of the loud thunder. \"Don't worry, Bob. I will find your home too.\"\nSuddenly, something unexpected happened. A bird flew near a window and landed on Bob's too. They became best friends. And from that day on"
+      ]
+    },
+    "eostok_apr": {
+      "temperature_0.6": [
+        " park to the park.\nAt the park, they saw a big ball. They played and, but then something unexpected happened. Max picked up the ball and started to play a lot of toys. They had a lot of fun together. Tim and Max learned that sometimes it was not good to be good friends.\n",
+        " train named Timmy saw a little girl named Lily crying. Her toy was stuck in a tree. Timmy stopped and said, \"Don't worry worry, I will help you.\" He picked up the ball and gave it to the little girl. The girl was very happy and thanked Tim for his help. They played together and",
+        "Once upon a time, there was a soldier who loved to march. One day, he went to the park to dance.\nIn the park, the clown met a little girl. \"Wow!\" she said, \" recognize it. \"I like your mustache.\" They became very good friends and they danced together.\n",
+        ". They loved to play together.\nOne day, the girl and boy went to the park to play. Tim was happy and started to gain a new friend. But soon, it was time for Tim to go home. They said goodbye and said goodbye. The girl and boy became best friends. They played together every"
+      ],
+      "temperature_0.75": [
+        " the park to play.\nAt the park, Tim found a toy. He played with friends. But something unexpected happened. Tom's friends started to smile and started to play with of the toys. They had a lot of fun together. Tim and Sam learned that it was always not good to be good friends.\n",
+        " station, Tom saw a little girl named Lucy crying. Her toy was stuck in a tree. Tom stopped and said, \"Don't worry Lucy, I will help you.\" He picked up the ball and brought it back to Amy girl. She was very happy and she thanked Tom for his help. They thanked Tom for",
+        "Once upon a time, there was a soldier who loved to bounce. One day, he went to the park to dance.\nIn the park, the giant soldier met a girl. \"Wow!\" she said, \"I like wind!\" They liked it and too. They became good friends and played together every day.",
+        ". They loved to play together.\nOne day, the bug and the game would play a game. Twinkle was very happy and started to gain a new friend. But soon, it was time for Ash to go home. They said goodbye and said goodbye. Twinkle and the bug became best friends. They played together every"
+      ],
+      "temperature_0.9": [
+        " the park to play.\nAt the park, they saw a big dog! Spot and his friends were scared of the dog. But then, the big dog started to play. It played with Spot and they all a lot of fun. Sam and his friends learned that it was not good to be good friends.\n",
+        " named named Tom saw a little girl named Amy. Her tight toy was stuck in a tree. Tom stopped and said, \"Don't worry worry, I will help you.\" He picked up the toy ball and gave it back to Amy. She was very happy and hugged Tim for her tight. She thanked Tom for helping",
+        "Once upon a time, there was a cricket who loved to bounce. One day, he went to the park to dance.\nIn the park, the cricket met a big dog. \"Wow!\" said the cricket. \"I like it too!\" he said. They became very good friends and played together every day.",
+        ". They loved to play together.\nOne day, the girl and the girl would play a game. Mia was very happy and started to gain a new friend.\nAfter it was time for Mia to go home, she said goodbye and said goodbye. Mia and the the girl became best friends. They played together every"
+      ],
+      "temperature_1": [
+        " park to the park.\nAt the park, they saw a big dog! Tim and his friends were happy to see it. But then, the big dog started to play with them. They played ball and a lot of fun together. Tim and his friends learned that it is not good to be good friends.\n",
+        " a time, there was a kind girl named Lily. Lily had a big, happy heart every day.\nOne day, the angel saw a little girl outside a garden. She picked up the ball and gave it to the little girl. The girl was very happy and hugged the angel.\nLily felt good for helping",
+        "Once upon a time, there was a who loved music. One day, she went to the park to play her violin.\nIn the park, judge met a dog named Max. \"Wow!\" she said, saying \"I like it and love it too. They became very good friends and played together every day.",
+        ". They loved to play together.\nOne sunny day, Tom and the bug played a fun game. They were both very excite to and meet a new friend.\nAfter that it was time for Tom to go. They said goodbye and said goodbye. Tom and the little spy bug became best friends and played together every"
+      ],
+      "temperature_1.1": [
+        " the park to play.\nAt the park, they saw a big dog! Spot and his friends were scared of the dog. But then, the big just started to play. They played with Spot and had a lot of fun. In the end, Fluffy learned that it is not good to be good friends. The",
+        " He lived in the woods a very kind girl named Lily. Lily had a kind, blue heart each day.\nOne day, Bob saw a little girl crying in the woods. He picked up the bunny and gave it back to her. The girl was very happy and hugged Bob for his help. Bob felt good because",
+        "Once upon a time, there was a clown who loved. Every day, the clown went to the park with his mom.\nAt the park, the judge saw a little girl. \"Wow!\" she said, saying \"I like name,\" it liked too. They became very good friends and they laughed and played together",
+        ". They loved to play together.\nOne sunny day, Tom and harmless bug played a fun game. They were both very excite excited to meet a new friend.\nAfter that it was time for Tom to go. They said goodbye and said goodbye. Tom and the little bug became their best friends and played together every"
+      ],
+      "temperature_1.25": [
+        " the park to play.\nAt the park, they saw a big dog! Buddy and friends were afraid about the dog. But then, the big dog came started to play. They played with Spot and had a lot of fun together. In the end, Tom learned that it was good not to be good friends.",
+        " He lived in the woods a very kind girl named Lily. Lily had a kind, blue heart each day.\nOne day, Bob saw a little girl crying in the woods. He picked up the bunny and gave it back to her. The girl was very happy and hugged Bob for his help. Bob felt good because",
+        "Once upon a time, there was a colorful judge. He loved to sell things He walked to the park with his mom.\nIn the park, a little girl looked at judge. \"Wow!\" she said, saying \"I like theers too\" and said. They became good friends and they laughed and played together",
+        " would talk and play every day.\nOne sunny day, the bug and the Tim played a catch game. They were both very excite to meet a new friend.\nAfter it was actually time to go home, patted the bug said goodbye and said goodbye. The bug and the bug became best friends and played together every"
+      ],
+      "temperature_1.5": [
+        " to the park.\nAt the big store, they saw a big toy! Sam's friends were there and excited at all. But then, Tim all all started to play. They played with Tim and had a lot of fun together. In the end, Sam learned that it was not good to be friends. to",
+        ". He lived in a kind named Lily. Lily had a little voice with a big, happy heart.\nOne day, the giant saw a crying speaking near a big. Lily picked the little girl's and gave it back to her. The girl was safe and felt happy for her family. Lily felt good that she",
+        "\nOnce there was wise stranger who liked to observe things. He would to slowly to the park and come with his mom.\nIn the park station a doctor looked up at Lucy. \"Wow!\" she said, \"I like the tree\" and smiled too.\nThey became good friends and they talked and laughed together",
+        " The cat loved to play too.\nOne day, the bird and Fluffy found a game to play. They were both very excited to go meet a new friend.\nAfter the end, it was time to go home. combed and Fluffy said goodbye. The girl and the cat became best friends and played together until the"
+      ]
+    },
+    "detached_apr": {
+      "temperature_0.6": [
+        " laugh a lot. One day, he saw a big hill and wanted to climb. But then, something unexpected happened.\nWhen when Tim climbed the hill, he found a dead apple. Tim was sad and angry \"This is not my toy!\" he said. His mom laughed and said, \"Don't worry,",
+        " Mia. Mia liked to outside in the sun. One day, she saw a big tree and wanted to climb it.\nMia went fast to her velvet dress. She was very happy. But then, something unexpected happened. Her dress got tight, and she could not play dress anymore. Mia was sad. She wanted",
+        "Once upon a time, there was a big red ball. The ball was very enthusiastic. It loved to play its friends in the park.\nOne sunny day, the ball rolled and something strange. The red ball turned into beautiful butterflies! They were blue alive. The kids were surprised and happy. They laughed and played",
+        " the magic flower every day.\"\nThe next day, the flower came to the garden. But something unexpected happened. The magic flower started to laugh. The flower said, \"I laughed, andYou will laugh too!\" The flower smiled and laughed, everyone smiled. And from that day on, Lily and the magic flower"
+      ],
+      "temperature_0.75": [
+        " he he outside. One day, he saw a big tree and ran to it, looked up and said, \"Look, look!\"\nLily looked at the tree and admired the sky. It is not scary,\" she said. Her father smiled and said, \"Don't worry, don't worry, don worry.",
+        " Mia. She loved to play outside and outside. One day, she saw a big tree and wanted to climb it.\nMia went fast to her velvet dress. She was very happy. But then, something unexpected happened. Her dress got tight, and she could not move anymore. She was sad again. She wanted",
+        "Once upon a time, there was a big red ball. The ball was very enthusiastic. It loved to play with friends in the park.\nOne sunny day, the ball rolled, something strange. The red ball turned into beautiful butterflies! They were blue! The kids were surprised and happy. They all played and laughed",
+        " with the king next day.\"\nThe next day, the bird came to the kingdom. But, something unexpected happened. The dog started to sing and dance! The king laughed and said, \"You are better too, bird!\" The bird thanked them and smiled. And from that day on, the people in the kingdom"
+      ],
+      "temperature_0.9": [
+        ". They liked to jog around the big forest every day. One day, Am L Landand above--edw area. \n\"Look, Johnny!\" Laz said. \"This big one looks looks fun!\"\nAlice said, \"Don't worry!\"\nDon't worry, \"Don't worry",
+        " the east. Mia liked to play in outside. One day, she saw a big tree and wanted to destroy it.\nMia wanted to make her friend happy. But, something happened.\nThe tree was not there! Her home was gone, and she could not her friend anymore. Mia was sad. She wanted",
+        "Once upon a time, there was a big brown dinosaur. The dinosaur was very friendly. He liked to make kids play and have fun.\nOne day, the kids heard the \"thank you\". They said, \"We dare you to play to!\" The kids were very happy. They played and laughed and played",
+        " with the king every day.\"\nThe next day, the girl tried to record the lion. But something unexpected happened. The dog started to sing and dance! The king laughed and said, \"You can sing well too!\" The bird too smiled. And from that day on, the girl and king became good friends."
+      ],
+      "temperature_1": [
+        ". She liked to ride around the big park day. One day, Nobody noticed it had Landand-fortedw balloon. \n\"Mom, look!\" pointed the balloon said \"This one is dangerous!\" she said. \"It looks like flying!\" mom smiled and said, \"Don't worry, sweetie",
+        ". Sally liked to play outside in the sun. One day, she saw a big tree and wanted to destroy it.\nSally wanted to make her mommy happy. But, something happened.The tree was not there! The sunflower everyone was was sad. Poor was not happy and she was not happy anymore. She lived",
+        "Once upon a time, there was a big brown there. This dinosaur was very friendly. He liked to make kids play and have fun.\nOne day, the dinosaur heard \"thank you\" to the bravest. They replied, \"We want to play together!\" The kids were happy. They played and laughed and played",
+        " with the bear next time.\"\nOne day, the bird tried to answer the fox. But something unexpected happened. The squirrel started to talk and laugh too. The bear laughed and said, \"You can sing too!\" The fox smiled and smiled. And from that day on, the girl and the squirrel became good friends"
+      ],
+      "temperature_1.1": [
+        ". She liked to ride around the park every day. One day, noticed upon it was a Land- upww balloon. \n\"Mom, look!\" pointed the balloon said \"This one is dangerous!\" she said. \"It looks like threat!\" Her mom smiled and replied, \"Don't worry,",
+        " house. Sally liked to play outside in sun. One day, Lily felt sad. She wanted to make a new house.\nLily to make her mommy happy. But,, something was not. Her grass was bad rain! Her mommy was sad. Poor blanket was not. Lily was not happy anymore. She was",
+        "Once upon a time, there was a big brown there. This dinosaur was very friendly. It liked to make kids play and have fun.\nOne day, the dinosaur said \"thank you to the kids play. They replied, \"We want to play together!\" The kids were happy. They played and laughed and played",
+        " to the show the day.\"\nThe day, the bird tried to hang the bald lion when something unexpected happened. It started to sing paws and laugh. The wise owl laughed and laughed. He said, \"I like to laugh like, bird!\" And from that day on, the monkey and the bird became very good"
+      ],
+      "temperature_1.25": [
+        " liked to play with everybody around the park every day. One day Lillie noticed it was aand- blast upww balloon. \n\"Mom, look!\" Layla called. \"This balloon one dangerous!\" she said. \"It looks like threat!\" Her mom smiled and replied, \"Don't worry,",
+        " sky. Sally liked to play outside in sun. One day, Lily felt nervous. She wanted to hide her makeup. Sally wanted to make Mom happy.\nBut when she was done, something unexpected was there. Her makeup gone! Everyone was sad. Sally makeup was not. She was not happy anymore. She was",
+        "Once upon a time, there was a big red ball. The ball was very obedient. The ball loved to make kids and have fun.\nOne day, a ball said \"university\" to kids. The ball replied, \"We want to play together!\" The kids were happy. They played and laughed and played",
+        " to the show the day.\"\nThe day, the bird tried to hang the monkey when something unexpected happened. The monkey started to sing its tweet silly songs. The owl laughed and laughed. He said, \"I want to dance and to help you.\" And from that day on, the monkey and bird became very good"
+      ],
+      "temperature_1.5": [
+        " liked to play contro around the park, zipping by went. One day Tim noticed it was big. Sorfoneess passed by.\n\"Mom, there, there an ambulance!\" Tim happily. \"I see a tornado!\" \"It looks like fog!\" His mom smiled. \"Don't be scared, Tim",
+        " Sally. Sally liked to play things down outside. One day, Lucy felt nervous. She wanted to hide her makeup. Sally wanted to make Mom happy.\nBut when Mia was done, something changed. The colors were gone. Her frown was gone. Mia was not happy. She was not happy anymore. She was",
+        "Once upon a time, there was a big red ball. The ball was very obedient. The ball wanted to play kids and have fun.\nOne day, it said ball \"un\" to kids. They said to ball, \"We want to play together!\" The kids were happy. They laughed and were excite.",
+        " nice to the the day.\"\nThe day, the bird tried to bite the old bear when something unexpected happened. The little bird turned into pretty things. The bald bear laughed and laughed. He said, \"I want to be friends you!\" And from that day on on, the girl and magic bird were magic."
+      ]
+    },
+    "mstok_v1": {
+      "temperature_0.6": [
+        " time, there was a little chicken. He lived in a house with a family. They liked to play and jump around every day. One day, they went outside to play. The sun came out the next day and they went outside to play. They soon became best friends. And they lived happily after.\n",
+        " loved to play together in the sun. One day, the bracelet broke Spot and it broke.\nLily was sad, but she had an idea.\nTom put the lotion on Spot's paw and something unexpected happened. The lotion made them they became smaller! They laughed and played all day long. Lily and Tom were",
+        " time, in peaceful town, there was a big bear. The bear was very nice and liked to play. But something unexpected happened. The unexpected bear happened, but something unexpected happened.\nBut then, something unexpected happened. The bear and the bear talked! They became best friends and played together every day. The bear",
+        " friend, a little girl named Mia. They loved to play with every day. They would every day.\nOne day, Mia and Tim found a big, colorful ball. Mia was very happy. She said, \"Thank you, Spot!\" They played with the ball and had fun all day. They had a great"
+      ],
+      "temperature_0.75": [
+        " intelligent dog named Max. Max liked to play, run, and to Tim play with. They would play and have fun every day. One day, they went outside to play. Tim went school next day, Max went dog again. They became best friends and became best friends. And they lived happily after.\n",
+        " day, he polished the day. One day, the beds played outside, and bed broke.\nTom was sad, but he had an idea.\nTom put glue on magic bed bed. Suddenly, something unexpected happened. The toys changed too! Now, they were alive. They laughed and played all day long.",
+        " time, in gloomy town, there was a big bear. The bear was very sad and he started to cry. The next day, something unexpected happened. But then, something something unexpected happened. The surprise!\nThe was a funny frog! The bear and became best friends. They played together every day and not every",
+        " friend, a little girl named Mia. They loved to play with every day. They would play day.\nOne day, Mia and Tim found a big, colorful ball. Mia was very happy. She said, \"Thank you, Spot!\" They played with the ball and had fun. They all day long. Mia"
+      ],
+      "temperature_0.9": [
+        " intelligent dog Max. Max liked to run and play with Lily. Lily liked Max too. They would play and have fun every day. One day, they went outside to play. Lily went to school, and Tim arrived again. Max was very happy. They had fun together. And they lived happily after.\n",
+        " day, he polished the day. One day, the drums played outside. The toys broke.\nTom was sad, but he had an idea.\nTom put glue on magic bed bed. Suddenly, something unexpected happened. The toys changed too! Now, they were happy and danced. They danced all day long.",
+        ". Inside the hole, there was a big cat inside. The cat was not scared and started to cry again. The cat did and something unexpected happened. But then, something unexpected unexpected. The cat stopped!\nThe big frog was magic! The cat and became best friends. They played together, every day not every",
+        " friend, a little girl named Mia. They loved to play outside every day. They would every day.\nOne day, Tim and Mia found a big, pink ball. Tim was very happy. \"Thank you, Mia!\" said. They played together with the pink ball had fun. They all day long.\n"
+      ],
+      "temperature_1": [
+        " intelligent dog Max. Max liked to run and play with Lily. Lily liked Max too. They would play and have fun every day. One day, they went outside to play. Lily went to school, and Tim arrived again. Max was very happy. They had fun together. And they lived happily after.\n",
+        " day, he shared the day. One day, the weather came outside. The toys broke.\nTom was sad, but he had an idea.\nTom put glue on old bed. Suddenly Suddenly, something unexpected happened. The toys changed too! Now, they were happy played outside. They played all day long.",
+        " Inside the hole there was a small, a big frog. The cat was not happy and started to cry again. The cat did and something unexpected happened. But then, something unexpected happened. The Tree\"!\nThe big frog was magic! The cat and became best friends. They played together, every day not every",
+        " friend, a little girl named Mia. They loved to play outside every day. They would every day.\nOne day, Tim and Mia found a big, pink ball. Tim was very happy. \"Thank you, Mia!\" said. They played together with the pink ball had fun. They all day long.\n"
+      ],
+      "temperature_1.1": [
+        " intelligent dog Max. Max liked to run and play with Lily. Lily liked Max too. They would play and have fun every day. One day, they went outside to play. Lily went school, and went to bed again. Max was very happy. They had fun together. And they all lived happily.\n",
+        " day, he made mattress time. One day, the football went outside. The toys broke.\nTom was sad, but he had an idea.\nTom put the sun on the bed. Suddenly, something unexpected happened. The toys changed too! Now, they were happy played outside. They played all day long.",
+        ". Inside the hole, there was a small green hat. It was a frog. Tim looked at the puppet and said, \"Ribbit, I like this frog, you!\"\nBut a fun thing happened. The frog became a frog! The talking frog and Tim were friends. They played together, every day not every",
+        " friend, a little girl named Mimi. Lily loved to feed Mimi every day. They would moo every day.\nOne day Mia and Mimi found a big, colorful ball. Mimi was very happy. \"Wow! I miss you!\" he said. Mimi played with the ball had fun. They all day long.\n"
+      ],
+      "temperature_1.25": [
+        " intelligent dog Max. Max liked to run and play with. Lily liked school Lily. They would play and have have fun every day. One day, they went outside to play. Lily went went to school,, and lay next to her. They had a fun day together. And they all lived happily.\n",
+        " day, he shared the time. One day, the football are rocked outside. It broke.\nTom was sad, but he had an idea.\nTom wore the sun on and something unexpected happened. The equipment could skip! Tom was surprised too, but they were happy and drew together all day, long and long",
+        " there was a playroom, there was a small green hat. It was a frog. Tim looked at the puppet and said, \"Ribbit, I like this frog, you!\"\nBut a fun thing happened. The toilet became a frog! The talking frog and Tim were friends. They played together, every day not not",
+        " friend, a little girl named Mia. Every day, they would play with Mimi and all day long.\nOne day, Mia and Bobo found a big, shiny ball. Mia was very happy. \"Mom! Mimi I miss it!\" She and Bobo played with the ball having fun. They all day long. Sometimes"
+      ],
+      "temperature_1.5": [
+        " restless named Max. Max liked to run and play with Lily. Lily liked Max lot. She would play with Tom every day. One day, they went outside to sleep again. Lily went to school, Max Max lay down next to her. They had a fun day together. And they all lived happily.\n",
+        " sky to the window all day. One day, the weather was outside. Tom was sad again. Tim saw was sad and had an idea.\nTom wore the sunglasses on the sky. Suddenly, the wind blew loud music again!\" Everyone was surprised, but they were happy. They played outside all day and had long",
+        ". Inside was a puppet, there was a small hat. It was a burrow. Tim looked at the puppet and said, \"He,, like this frog, you!\"\nBut a fun thing happened. The crackers frog changed! Tim and Tim talking friend laughed. They were together and friends all day long. One",
+        " friend, a little girl named Mia. Every day, Tim would come to the bedroom to next play.\nOne day Mimi and Tim found a big, shiny ball inside. Tim was so happy! \"Fluffy, threeife miss it!\" She and Bobo played with the ball having fun. They all day long. Sometimes"
+      ]
+    }
+  }
+}

ts-fivearm/evaluation/genppl-hf-gpt2-large-512-five-arm-v2.json

@@ -0,0 +1,341 @@
+{
+  "protocol": {
+    "reference_model": "gpt2-large",
+    "n_samples": 512,
+    "seed": 42,
+    "n_provided_levels": 0,
+    "level0_condition": "learned BOS query",
+    "prompt_tokens": 0,
+    "implementation": "evaluate.load('perplexity', module_type='metric')",
+    "add_start_token": true,
+    "aggregation": "arithmetic mean of per-example perplexities",
+    "sampling": {
+      "argmax": {
+        "temperature": 1.0,
+        "top_k": 1,
+        "top_p": 0.0
+      },
+      "top_k50_p095": {
+        "temperature": 1.0,
+        "top_k": 50,
+        "top_p": 0.95
+      },
+      "random": {
+        "temperature": 1.0,
+        "top_k": 0,
+        "top_p": 1.0
+      }
+    }
+  },
+  "metadata": {
+    "two_stage": {
+      "checkpoint": "outputs/ncm/tinystories-two-stage-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "two_stage"
+    },
+    "naive_e2e": {
+      "checkpoint": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "naive_e2e"
+    },
+    "eostok_apr": {
+      "checkpoint": "outputs/ncm/tinystories-eostok-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "eostok_apr"
+    },
+    "detached_apr": {
+      "checkpoint": "outputs/ncm/tinystories-detached-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "detached_apr"
+    },
+    "mstok_v1": {
+      "checkpoint": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
+      "step": 82813,
+      "levels": [
+        1,
+        4,
+        9,
+        16,
+        25,
+        36,
+        49,
+        64
+      ],
+      "training_regime": "mstok_v1"
+    }
+  },
+  "results": {
+    "two_stage": {
+      "argmax": {
+        "hf_mean_perplexity": 13.720980644226074,
+        "median_example_perplexity": 13.720980644226074
+      },
+      "top_k50_p095": {
+        "hf_mean_perplexity": 46.59727036766708,
+        "median_example_perplexity": 40.54710006713867
+      },
+      "random": {
+        "hf_mean_perplexity": 119.348518460989,
+        "median_example_perplexity": 101.71148681640625
+      }
+    },
+    "naive_e2e": {
+      "argmax": {
+        "hf_mean_perplexity": 70.57276153564453,
+        "median_example_perplexity": 70.57276153564453
+      },
+      "top_k50_p095": {
+        "hf_mean_perplexity": 94.89091111533344,
+        "median_example_perplexity": 51.998064041137695
+      },
+      "random": {
+        "hf_mean_perplexity": 129.14280272647738,
+        "median_example_perplexity": 108.94772338867188
+      }
+    },
+    "eostok_apr": {
+      "argmax": {
+        "hf_mean_perplexity": 21.72423553466797,
+        "median_example_perplexity": 21.72423553466797
+      },
+      "top_k50_p095": {
+        "hf_mean_perplexity": 37.74269070290029,
+        "median_example_perplexity": 33.8486213684082
+      },
+      "random": {
+        "hf_mean_perplexity": 129.28525669127703,
+        "median_example_perplexity": 109.42802810668945
+      }
+    },
+    "detached_apr": {
+      "argmax": {
+        "hf_mean_perplexity": 16.042163848876953,
+        "median_example_perplexity": 16.042163848876953
+      },
+      "top_k50_p095": {
+        "hf_mean_perplexity": 45.35750689730048,
+        "median_example_perplexity": 39.72639846801758
+      },
+      "random": {
+        "hf_mean_perplexity": 144.0669886134565,
+        "median_example_perplexity": 119.7023811340332
+      }
+    },
+    "mstok_v1": {
+      "argmax": {
+        "hf_mean_perplexity": 18.070493698120117,
+        "median_example_perplexity": 18.070493698120117
+      },
+      "top_k50_p095": {
+        "hf_mean_perplexity": 44.25286133028567,
+        "median_example_perplexity": 39.68633270263672
+      },
+      "random": {
+        "hf_mean_perplexity": 138.12906496599317,
+        "median_example_perplexity": 113.1609115600586
+      }
+    }
+  },
+  "samples": {
+    "two_stage": {
+      "argmax": [
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a big dependable dog. One day, the dog went to to find a friend.\nAt the park, they met a friend. The friend was very happy and said, \"Thank you, dependable dog! You are a good friend.\" And they lived happily ever after.\n"
+      ],
+      "top_k50_p095": [
+        " there was no treasure there. In the park, there was a big, shiny.\nSue took the rock home, and shared it with her dad. They were all so happy that that it is important to help others. Tim learned that helping others can make him happier too. The moral of the story is that when",
+        " letter on the ground. She noticed it lying on the ground. She picked it up. The peanut was very happy. The little package belonged to a little boy named Tim.\nTim smiled and said \"thank you you, Lily! You found my peanut.\"\nThen, something unexpected happened.\nAs the peanut,",
+        " friend, a cat her friend came to play. The toy touched her toy. \"Oh no!\" She said. \"Ouch! This will hurt, hurt you, and I will get you soon.\"\nThe cat and the toy played together all day. They laughed, talked, and laughed. They played all day,",
+        " it was happy.\nOne sunny day, a boy and a squirrel went to the park. It saw a big tree and wanted warn him to be careful. \"Be,\" said the squirrel, \"but the tree is nice, but it is important.\" The boy learned to listen to the others and help others when he",
+        " little boy named Tim. Tim liked the moon and wanted to see the sunrise.\nThe rocket said, \"I want to see a night star tonight.\"\nThe boy looked at the rocket in wonder, and thought disagreed. \"I just don't guess I think it was so interesting either.\" \nSo, they",
+        " a big black cat. It lived in a small house with a family. The cat loved to play and eat mice every day, the bird would give the rare cat yummy food. But then, something unexpected happened. The unexpected happened! The rare cat was very surprised. The family laughed, too. They became best friends",
+        "The little girl smiled Fluffy and gave Amy a new toy. It was a small dog, with a red heart. Lily knew the dog friendly. They played together all day. When it was time for Anna to go home, she it was inside.\nFrom that day on, Amy and the dog were best friends.",
+        " met a big ant, ant met little train named Tim. He asked, \"Andy, will you help me today?\" Andy said, \"Yes I, I will help you carry.\" Tim was happy to help.\nThe ant and Tim became good friends. They talked good and became good. The family ant was happy"
+      ],
+      "random": [
+        " a rock.\nEvery day he started went to the park day and played a lot. After a while it saw a very thin pole. He was excited to see up and he could see the top of the practiced pole it went. The child and dove dove pole. This person made it so curious carefully.\nThe",
+        " so he began to move to the side of the drum. When he looked the way, he saw a girl crying. She lost her toy. The girl was very happy and said, \"Thank you, Tom!\" Tom felt good because he could use new drum to give a toy. He pushed a big smile on the",
+        " demanded. A Ay friendly cricket was eating beef beef when Bunny's mud.\n\n\"Oh yummy! Can I help?\" asked Bunny.\n\"Sure please,\"oh said. \"The salt made the pile popcorn. Bunny jumped into Splash! He was happy when he was done and he felt full full.\n",
+        " play with you!\"\nThe small bird and Oliver decided to play with the tree. But the small tree had to push it a round way home. The small tree did not understand how to split down. The small bird did not know how to scared. It pleaded \"Please, I am bright a wall and I am",
+        " very big and wide. Her mom pointed to a stick and touched the star. The magic wand was coming to was big, strong dragon flying.\nThe stick was at the fog at. The dragon was big and looked far away. The dragon flew away to Amy her mom and dad.\nAmy told her mom about",
+        " a toy. It was a popular toy with a chain a magic wand, pulled an dinosaur. And, the child, Zero, trapping it around. Suddenly, Lisa noticed a small kite with wooden on the ground. She thought it looked special.\nCan we play with it?\" Tom asked quietly, quietly.\n",
+        "ination, the nail.\nDan was very stubborn and said, \"No certain! You do it again!\" Listen and the others and again the phone Tom did and said, \"Tom, you wipe the slide clean\". a qu out the nail and he started wiping the table. Then, he put the paper down down",
+        "\nBut the bullyally was being sending the bird too tight. Together, they said, \"Please help you!\" They both replied, \"Yes, I do. They.\"\nBut then, the bird said. \"Then something happened.A little bird. I can birds come flying towards me. I am talking to"
+      ]
+    },
+    "naive_e2e": {
+      "argmax": [
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s",
+        " was a little girl girl called Nan. She was only three years old and very very curious.\n\u201cWhat\u2019s?\u201d said Jan. \u201cLets see what\u2019 it.\u201d \nJananan noticed the Holly\u2019ss shoulder. \u201cWhat\u2019s that,\u201d she said. \u201cLet\u2019s"
+      ],
+      "top_k50_p095": [
+        " girl replied, \"Would you like a joke?\"\nThe rabbit smiled and said yes. The old man nodded and smiled. \nThe girl smiled. She realized she didn't understand why still. Then the girl asked the man \"Why did you complain too?\"\nThe old man smiled and said, \"It was",
+        ", and the voice said that it was so upset. \nShe was so ignorant that she just couldn't believe that she died. She didn't think she forget the mean witch. \nThe nightmare was gone and he slowly disappeared away. \nThe little girl's heart were reappfes again. She",
+        " they arrived at the bakery arrived at school weekend.\nWhen he arrived, he saw the oat he was so excited to try it. He was so excited! He got some of sweets chocolates. \n\n \nThe children loved both kinds of delicious delicious waffles. \nThe family started licking and started eating their",
+        " In this town, there lived a small girl named Sue. Sue was scared of the storm. Tom said, \"Don't worry, I will find your home.\"\nJust then, something unexpected happened. A bird flew into a town and landed on Sue bed too. They became became best friends. From that day on",
+        " all day, shooting pretty sunset. One day, she built a big house outside with her friends.\nOne sunny day, Lily met another new friend, named Lily. Lily said, \"Hi, I am Poppy. I like your colors. Let's dance together!\" And And from that day, they celebrated all celebrated",
+        " little house, there lived a boy named Tom. Tom loved to stretch big in the sun. One day, he saw a big suit in a shop. The rich man was very happy but but too surprised.\n\"Hello, lady!\" said Tom. \"Hello, can meet?\" asked Tom. \n\"Yes",
+        " \u201cIt\u2019s perfect,\u201d the mummy replied, \u201cIt\u2019s shiny.\u201d \nSuddenly, the daughter noticed that something else, not the same as something else. \nThe girl and the little girl then ran over to his mother daughter. She smiled and said, \u201cLike?\u201d \nThe daughter said and",
+        "imimed and pluged to the edge of the lake. \nDong! \nSuddenly he saw something sparkly in the water. He noticed something chatterAliceing down hisness and he saw it pearl sparkle! It was alive! He was so excited. \nThe frog was so happy that he"
+      ],
+      "random": [
+        " big bear named Timmy saw how close to teach everyone everyone was saw Tweet and liked to show his friends's style. One girl called out, \"Please give me of us show, why don't?\"\nBut Tweet didn come over. Tweet said, \"I want to display you and show our things for everyone.\" Timmy",
+        " I done!\" She hugged his kissed and upset him. She said, \"There, that we want something else. Maybe we need in need. We are pretending to be nice. Get much than yourself.\"\nBen felt heart. He felt his tummy hurt. The dream was now now he was now real. He thanked",
+        ". They went to the east, their animal friends started to cry. But, there was a big, dark bird. The bird was still not what she had calling about, but it was not either harmless. So, Blue told a joke because he was sad because he couldn't happy. Spot happy was not talk to",
+        " bird and can not fly it.\nTim wanted to help Bella, so he took it to her mom tomatoes. But they had however at the same strip with Tim and his egg! \"Ow, squeezing, Mom That apple is sour. Like my mouth hurts.\" His mom hugged him and said, \"It's okay",
+        " something that he needed to wait when he came to come in. As he played, Max found a question about Max's ball. The man was hiding behind the ball.\nAs Max looked, he found the hidden box. He brought the box and knew it was his ball! The dog had come to play with Sue",
+        " They saw a big cat near the water. The cat was giving to give Spot a massage. Sue wanted to help, but Amy she was deaf so she couldn't reach it.\nLola started to whistle a happy song. Spot was surprised but happy. He wagged his tail and played the wall for rhythm for the cat",
+        " When the noise had vanished empty, John was sad and his petured was getting hungry.\nThen, something broken white something boy named Jack stamped inside the field. The little boy's smile in his hand, he saw to fix his shine look and said \"Leared, zebra is ext what is ignorant?\".\nThe",
+        " scooter to play outside. It was a nice day at school. Sue took her scooter to play park with other kids. She saw many kids playing. Sue wanted to play. Bob's cube. She asked her friend if she could borrow interest's football back. Sue's owner said yes named Chirp.\nSue was so"
+      ]
+    },
+    "eostok_apr": {
+      "argmax": [
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n",
+        " friend for her new friend. She asked her mom, \"Can I my friend be my friend?\"\nThe next day, the little bird came to the castle. The girl hugged her and said, \"Don't worry, we will forgive. We can be friends.\" And they all lived happily happily ever after.\n"
+      ],
+      "top_k50_p095": [
+        " park to the park.\nAt the park, they saw a big dog! Tim and his friends were happy to see it. But then, the big dog started to play with them. They played ball and a lot of fun together. Tim and his friends learned that it is not good to be good friends.\n",
+        " a time, there was a kind girl named Lily. Lily had a big, happy heart every day.\nOne day, the angel saw a little girl outside a garden. She picked up the ball and gave it to the little girl. The girl was very happy and hugged the angel.\nLily felt good for helping",
+        "Once upon a time, there was a who loved music. One day, she went to the park to play her violin.\nIn the park, judge met a dog named Max. \"Wow!\" she said, saying \"I like it and love it too. They became very good friends and played together every day.",
+        ". They loved to play together.\nOne sunny day, Tom and the bug played a fun game. They were both very excite to and meet a new friend.\nAfter that it was time for Tom to go. They said goodbye and said goodbye. Tom and the little spy bug became best friends and played together every",
+        " hero named Tom. Tom liked to play in the sun. He would run and jump in the park.\nOne day, Tom found a in a tree. The cat was stuck. Tom was happy. He took the cat home and told the couple his cat friends. They were very happy and excited too. They became",
+        " were going to to the park and play.\nWhen they got to the park, Tom had to hear some news. He had an and Sam had an idea He pointed to to a sign and said \",4!\" The man looked at the cards and had the same thing Tom told him that a fun game that was",
+        " upon a time, there was a boy named Tim. Tim was a very naughty boy. Every day, he toys toys tasks. But then, something interesting happened. Tim wanted talking! He asked again, \"Why can't I use my imagination?\" The junk replied nothing.\nIn the end, the In In the",
+        "\n\nOnce there was a little girl named Molly. She loved smile at the billboard. He smile smile morning she would smile and \"Look at all this things I got here!\"\nOne day, Molly came across a very big billboard. It was huge and heavy. Molly was so curious. \nShe picked up"
+      ],
+      "random": [
+        ". \n\"Hi Bob, can you help me clean this bec!\" Jim said. Max excitedly nodded and they started started to hang all buckets of hill clean things.\nAfter they were all done, Jim Jim Mary took the mud workers into the factory and let them come over them how to count and how build.",
+        " Jack asked, hands holding a button. \" that said Whustr', he hurt.\nMommy said \"it could act like. Do you want flapping question?\" Jack thought before he could take deep eyfish look on his face. His face froze. \".\nMummy smiled and said, \"It's Tom Jojo",
+        " a toy car with chocolate: Bl favorite sandwich.\nMia and her red sandwich found a great spot under a big tree to eat. The more she saw was the sky pink pink and hot. Mia wore her yellow sandwich every day salad with her red looking at. But soon, a naughty boy took a bite of his",
+        " impressive to help Spring Pippin Baby held on tight to the carrot and kept going. Eventually, Big leaving Kiki kept chewing his had carrot\n morning they played, it got dark and Pippin knew it was time to go home. Pippin's warning and ran into a rock, dug a big hole.\nAndy put the carrot in",
+        " put it alone and it opened itself by. and Ben fell on the floor. Tim shut the door, so they pushed each other off.\nWhen they turned around, they found a big, cozy house in the corner. They went inside and Cat said \"I am cold,\" said Tim. \"I will help you",
+        " tree with lots of limes leaves. The cat wanted to use the tree.\nThe cat climbed up and looked at the tree. The cat said, \"Meow! That is sharp. You should not touch them.\" The girl knew the cat's words. The girl saw cat and said, \"Hey, not touch the",
+        " said: \"That's very nice, Lila. It is way from other trees later.\" So then she heard a loud noise. Cat was her was dad. He told Anna they should stay play in the forest and Remy's Fluffy should go and find out what is not nice for Katie, she decided to find out to",
+        " to look for it perfume. She looked everywhere for it. She was worried. \"Then we shouldn't be perfume. It might be in the drawer.\" It's no word word. \"Okay, I will open it,\"Maybe Amy repeated.\nMom looked at the picture with her big mouth muffins muffins \"Lily,"
+      ]
+    },
+    "detached_apr": {
+      "argmax": [
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n",
+        "\nOnce upon a time, there was a little girl named Lily Lily. Every day day, Lily talked every day. But one day, something unexpected happened.\nOne day something unexpected happened. Instead, Lily, and the cat became best friends. They played together every day. And they lived happily ever after.\n"
+      ],
+      "top_k50_p095": [
+        ". She liked to ride around the big park day. One day, Nobody noticed it had Landand-fortedw balloon. \n\"Mom, look!\" pointed the balloon said \"This one is dangerous!\" she said. \"It looks like flying!\" mom smiled and said, \"Don't worry, sweetie",
+        ". Sally liked to play outside in the sun. One day, she saw a big tree and wanted to destroy it.\nSally wanted to make her mommy happy. But, something happened.The tree was not there! The sunflower everyone was was sad. Poor was not happy and she was not happy anymore. She lived",
+        "Once upon a time, there was a big brown there. This dinosaur was very friendly. He liked to make kids play and have fun.\nOne day, the dinosaur heard \"thank you\" to the bravest. They replied, \"We want to play together!\" The kids were happy. They played and laughed and played",
+        " with the bear next time.\"\nOne day, the bird tried to answer the fox. But something unexpected happened. The squirrel started to talk and laugh too. The bear laughed and said, \"You can sing too!\" The fox smiled and smiled. And from that day on, the girl and the squirrel became good friends",
+        " was a little boy Tim. Tim loved to eat at the tree. He loved to go near the big olives near the garden every day. \nOne day, something strange happened. The flowers started to change and grow! Tim became candy Tim became rich! He shared the lemonade with his friends. They played and ate",
+        "Once upon a time, there was a girl named Lily. She was an ordinary snake named Lucy. She had an idea. She said, \"Let's play a game!\" The ugly snake looked at Lucy and said, \"Yes, let's play catch.\" Lucy agreed. They played all day, and the snake was",
+        " A boy named Tim came to play. The brilliant bird liked to watch Tim play. They loved to play together.\nThen something unexpected happened. The bird went the ground. The bird looked at Tim and said, \"Have game. Let's all play together.\" Tim took turns at catching the bird. They took turns",
+        " time, there was a little girl named Lily. She loved to play outside in the sun with her friends. One day, she found a new bird in the grass. bird She didn't know who but spoke to the bird, so they became friends. They played and laughed together. The crazy bird was not lonely anymore"
+      ],
+      "random": [
+        " very young person, who liked to watch by. Do lived near the counter. Every day, the patient questioned.\nOne experward, \"Where first more fun?\" But Jimsy said first, one \"Wait you push them down there!\", 'Yes, you can\" Paul replied, 'That's a great",
+        " Sue did not like the noise. Max did not like the marker with the loud. his, this language, and they started to walk again.\nAs they marched, they saw the ants to to plant tries to help. \"Hi Sue said to Bob. \"Who pointed food?\" they asked. Max said, \"",
+        " it on speak shake her up. Timmy saw the tail, and picked it up \", squirrel! I found a treat in the garden. It is so pretty and juicy to eat the tooth.\"\nBlue ran to find Timmy, who had been lost under the apple saw bird. It was big. \"I remembered a",
+        " time there was a little girl named Spot. Spot was brave and kept letting for walks. Whenever he was calm and nodded nod over. \nAs Spot walked, he saw a pile of power. He looked up and his out Kitty was scared of huge man! The stupid man had run away had lost the magical powers",
+        " \"Tom, thought that nice lady on the mysterious mask was. It said, \"The mask belongs we can play with and play in!\"\nThe lady put mask on her lunch and it shared Sue and Tom had dream fun. They opened their toys and yummy food. Kitty and Tom became good and stayed forever. They",
+        " couldn't stop jumping and laughing.\nAs they played, they saw close fires coming to thunder. Mia was scared because she had never been high from before from lightning up so high! But Max started to flap his wings and jumped flap up and Tim. Mia tried to fly, but they could not reach it. He",
+        " said he was not So Charlie agreed permitted and so he agreed. They started work until the delivery was honest man. The muscle was nice and said fixing how messy Fred was collected and he did things as Jack.\nThe could borrow Fred's nice things give. He was sad too, but he also knew he had to",
+        " and her friends have a smile one one home. She cuddled Lucy with a picture of a little boy, and Tim's friend, Miss Miss.\nEveryEvery day, Lily would blink up towards Wednesday and slide at night. The children would watch Wednesday rise. It wasken ror with a sword, Mr Dis"
+      ]
+    },
+    "mstok_v1": {
+      "argmax": [
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after",
+        " together in the park. One day, she found a hidden box in the park.\nOne day, Lily saw a small bug on the ground. She picked it up and looked at it. The bug looked Lily and said, \"Thank you, you promise to be my friend too.\" And they lived happily ever after"
+      ],
+      "top_k50_p095": [
+        " intelligent dog Max. Max liked to run and play with Lily. Lily liked Max too. They would play and have fun every day. One day, they went outside to play. Lily went to school, and Tim arrived again. Max was very happy. They had fun together. And they lived happily after.\n",
+        " day, he shared the day. One day, the weather came outside. The toys broke.\nTom was sad, but he had an idea.\nTom put glue on old bed. Suddenly Suddenly, something unexpected happened. The toys changed too! Now, they were happy played outside. They played all day long.",
+        " Inside the hole there was a small, a big frog. The cat was not happy and started to cry again. The cat did and something unexpected happened. But then, something unexpected happened. The Tree\"!\nThe big frog was magic! The cat and became best friends. They played together, every day not every",
+        " friend, a little girl named Mia. They loved to play outside every day. They would every day.\nOne day, Tim and Mia found a big, pink ball. Tim was very happy. \"Thank you, Mia!\" said. They played together with the pink ball had fun. They all day long.\n",
+        ", there was a small girl named Lily. She lived in a house with her neighbor Tom. One sunny day, they went for a walk in the woods. The woods was very cold, but Lily was very strong.\nAs they walked and talked, they became friends. They played and laughed all day. The sun",
+        " was a little girl, very excited. She started playing playing with her toys and suddenly she saw something. She said \u201cOh no, I isn\u2019t so fine!\u201d\nMommy\u2019s eyes and laughed.. \u201cIt\u2019s ok,\u201d they said.\nBut then Emily's mommy came into the kitchen. \u201c",
+        ", play. One day, Yesterday wanted to race with her dolls and race cars. She felt uncomfortable because she forgot everything.\nMom had an idea. She asked Jojo if she had some medicine. Anne didn't feel like it didn't yucky. Her mom was happy. Herson was not sad anymore. She was",
+        " upon a time, was a little bird who liked to go adventures together.\n\u201cI promise,\u201d said the frog. \u201cYes, you can please lead me too.\u201d \nAfterwards, the owl, and something unexpected happened. They became best friends. They played together in the big meadow and they were the best of"
+      ],
+      "random": [
+        " little sad, but was a little shy.\nOne sunny day, the green and the kitchen were outside. They danced and danced, making flowers and bugs. They sang and music. Lily was having so much fun.\nBut then, Lily did not hear her mom calling her to stop playing and come back. She",
+        " had to fly. He had a big smile on his face, smiling and to turn turn. He filled with his family and food with his friends. He stayed at the airport where his people helped who they needed help.\nThe family tried and soon they needed the help. He said goodbye. The little bird flew away",
+        " the toy. She wore the pretty dress to her friend, Pete's dress and anxious and was not gloomy anymore.\nAt last, both Pete Mousy and the thief share toys the enjoy at the post. They were so amazed to see the pretty queen cape that the hat was her. The unique job became even more beautiful",
+        " was a lively little cat named Fluffy. Fluffy loved to play with her friends, Yesterday. One day, while they were playing hide and seek, Fluffy found a big box. \"Let's open about it!\" said Fluffy. \"Let's try this and find what is inside.\" Suddenly, they croaked to find who",
+        ". She wanted to take the end of the garden. Mia moved the loop around the sun too.\nThe next day walked, van goes around the garden. Mia saw the ancient tree and tried to talk to it. It whispered, \"Lily, look the tree is so except!\" Lily laughed and Mia played all day",
+        " Robbie made music of a tea pot.\nOne day, Uncle saw a nice box. He opened it. She found a toy so much toy bear. Jack was very polite and said thank you to the furry bear and started to play.\nAll of the tea party, Jamie's Uncle Pete flew into the room and",
+        " lots of fun every day.\nOne sunny day, the cat and the dog wanted to have a fun game play near the shut. The cat said, \"Hurry friends, let's play!\" They played a guessing toy who could stuck or see \" wins\" and then to catch them.\nAfter a while,",
+        " big suit. Lily did not know what was here. Then, she played it all by himself. The animals who also liked to to play games with Tom. They all said, \"I played quiz with you!\" Tom laughed, She Lily.\nJust then, a cat named Sam winked. the Tom was not a"
+      ]
+    }
+  }
+}

ts-fivearm/evaluation/summary.md

@@ -0,0 +1,21 @@
+# TinyStories five-arm final evaluation
+
+## Fixed-policy GPT-2-large generation perplexity
+
+| Arm | Argmax | Top-k 50, p=.95 | Unrestricted |
+|---|---:|---:|---:|
+| two_stage | 13.7210 | 46.5973 | 119.3485 |
+| naive_e2e | 70.5728 | 94.8909 | 129.1428 |
+| eostok_apr | 21.7242 | 37.7427 | 129.2853 |
+| detached_apr | 16.0422 | 45.3575 | 144.0670 |
+| mstok_v1 | 18.0705 | 44.2529 | 138.1291 |
+
+## Entropy-matched GPT-2-large generation perplexity
+
+| Entropy (nats) | two_stage | naive_e2e | eostok_apr | detached_apr | mstok_v1 |
+|---:|---:|---:|---:|---:|---:|
+| 3.5733 | 35.6269 | 74.2362 | 27.0211 | 35.1928 | 32.7351 |
+| 3.5936 | 40.1734 | 80.5489 | 31.1786 | 43.2390 | 38.0498 |
+| 3.6140 | 45.2529 | 88.7783 | 38.3632 | 49.9481 | 42.9439 |
+| 3.6343 | 55.2194 | 99.5642 | 45.5228 | 63.9253 | 52.7307 |
+| 3.6547 | 68.6164 | 121.0413 | 59.6883 | 77.3142 | 69.3435 |

ts-fivearm/manifest.json

@@ -1,21 +1,4 @@
 {
-  "campaign": "tinystories-five-arm-v2-5epoch-ddp",
-  "code": {
-    "branch": "tinystories",
-    "commit": "522285b",
-    "repository": "HazyResearch/MsTok"
-  },
-  "global_batch_size": 512,
-  "huggingface_prefix": "ts-fivearm",
-  "initialization": {
-    "detached-apr": "scratch-seed-42",
-    "eostok-apr": "scratch-seed-42",
-    "mstok-v1": "scratch-seed-42",
-    "naive-e2e": "scratch-seed-42",
-    "two-stage": "pretrained-codec-plus-fresh-seed-42-ncp"
-  },
-  "steps": 82813,
-  "world_size": 4,
   "artifacts": {
     "codec": {
       "bytes": 384343801,
@@ -23,15 +6,63 @@
       "sha256": "63d7064395dce819bdb5306b3337f28acf2f5117ce120f7064e06c304850947c",
       "status": "complete"
     },
-    "detached-apr": {"path": "detached-apr/step-82813.pt", "status": "pending"},
-    "eostok-apr": {"path": "eostok-apr/step-82813.pt", "status": "pending"},
-    "mstok-v1": {"path": "mstok-v1/step-82813.pt", "status": "pending"},
-    "naive-e2e": {"path": "naive-e2e/step-82813.pt", "status": "pending"},
-    "two-stage": {
+    "detached_apr": {
+      "bytes": 1484530939,
+      "path": "detached-apr/step-82813.pt",
+      "sha256": "0c843d79642d1d8c0ef33b5853eddcf42313d3a178b304920e8f7cb0f42da1a1",
+      "status": "complete"
+    },
+    "entropy_matched": {
+      "bytes": 53388,
+      "path": "evaluation/genppl-entropy-matched-gpt2-large-512-five-arm-v2.json",
+      "sha256": "db2b6cbc206899e6584616cc986ecc6716305c3db13bb7eea412dce6ef7445a8",
+      "status": "complete"
+    },
+    "eostok_apr": {
+      "bytes": 1484530939,
+      "path": "eostok-apr/step-82813.pt",
+      "sha256": "d5229ce864096e636a94874978b25f2a2206058f9e165d5bee4084cec16093a2",
+      "status": "complete"
+    },
+    "fixed_policy": {
+      "bytes": 38698,
+      "path": "evaluation/genppl-hf-gpt2-large-512-five-arm-v2.json",
+      "sha256": "2394f59939caace4dcc8d3e07d4d069525db9f6431f26d1fd535fbe00555030a",
+      "status": "complete"
+    },
+    "mstok_v1": {
+      "bytes": 1747368555,
+      "path": "mstok-v1/step-82813.pt",
+      "sha256": "da58fca14a3483159036efa2f0edac2149051379ee2359fe29b4a2f249fda124",
+      "status": "complete"
+    },
+    "naive_e2e": {
+      "bytes": 1484530939,
+      "path": "naive-e2e/step-82813.pt",
+      "sha256": "68c479b62d102437a7fc668a0cd613d531e12000250652c0f9b89bee4c2918ae",
+      "status": "complete"
+    },
+    "two_stage": {
       "bytes": 1363101739,
       "path": "two-stage/step-82813.pt",
       "sha256": "c6e3e01cf0af8b880ef8374959d04a87b4e89d55aaae4795b520e534839dff99",
       "status": "complete"
     }
-  }
+  },
+  "campaign": "tinystories-five-arm-v2-5epoch-ddp",
+  "code": {
+    "branch": "tinystories",
+    "commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
+    "repository": "HazyResearch/MsTok"
+  },
+  "global_batch_size": 512,
+  "initialization": {
+    "detached_apr": "scratch-seed-42",
+    "eostok_apr": "scratch-seed-42",
+    "mstok_v1": "scratch-seed-42",
+    "naive_e2e": "scratch-seed-42",
+    "two_stage": "pretrained-codec-plus-fresh-seed-42-ncp"
+  },
+  "steps": 82813,
+  "world_size": 4
 }