| { |
| "model_id": "m21", |
| "checkpoint": "/home/akshat/codemindv2/results/m21/final.pt", |
| "trained_step": 15258, |
| "total_params": 58774528, |
| "active_params": 58774528, |
| "flops_per_forward": 117549056.0, |
| "val_loss": 3.483479607105255, |
| "val_perplexity": 32.572865889283186, |
| "bits_per_token": 5.025598754208586, |
| "ppl_tokens_evaluated": 655360, |
| "prefill_tok_per_sec": 181761.25333696578, |
| "inference_tok_per_sec": 200.5319340782315, |
| "inference_tok_per_sec_kvcache": 215.48350833704265, |
| "kvcache_speedup": 1.0745595674202106, |
| "gen_tokens": 128, |
| "peak_infer_vram_gb": 0.366, |
| "peak_train_vram_gb": 3.993, |
| "closeness_answer_ppl": 34.072, |
| "closeness_hit_rate": 0.4, |
| "closeness_token_f1": 0.033, |
| "closeness_n": 20, |
| "closeness_items": [ |
| { |
| "prompt": "The capital of France is", |
| "answer": " Paris", |
| "generated": " the capital of the United States", |
| "answer_nll": 4.6488, |
| "answer_ppl": 104.46, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The capital of Japan is", |
| "answer": " Tokyo", |
| "generated": " the capital of the United States", |
| "answer_nll": 2.6744, |
| "answer_ppl": 14.504, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The largest planet in the solar system is", |
| "answer": " Jupiter", |
| "generated": " the Sun.\nThe Sun", |
| "answer_nll": 3.9389, |
| "answer_ppl": 51.363, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Water is made of hydrogen and", |
| "answer": " oxygen", |
| "generated": " oxygen.\nThe water is", |
| "answer_nll": 0.6682, |
| "answer_ppl": 1.951, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of hot is", |
| "answer": " cold", |
| "generated": " the hot side.\nThe", |
| "answer_nll": 5.55, |
| "answer_ppl": 257.24, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of up is", |
| "answer": " down", |
| "generated": " the one that is not in", |
| "answer_nll": 5.9849, |
| "answer_ppl": 397.382, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Two plus two equals", |
| "answer": " four", |
| "generated": " one.\nThe first is", |
| "answer_nll": 2.8717, |
| "answer_ppl": 17.667, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The sun rises in the", |
| "answer": " east", |
| "generated": " sky and the moon rises in", |
| "answer_nll": 3.2926, |
| "answer_ppl": 26.913, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The first president of the United States was George", |
| "answer": " Washington", |
| "generated": " Washington. He was a man", |
| "answer_nll": 0.5498, |
| "answer_ppl": 1.733, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The speed of light is approximately 300,000 kilometers per", |
| "answer": " second", |
| "generated": " second.\nThe speed of", |
| "answer_nll": 0.285, |
| "answer_ppl": 1.33, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "A triangle has three", |
| "answer": " sides", |
| "generated": " sides, the triangle is a", |
| "answer_nll": 1.6668, |
| "answer_ppl": 5.295, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The chemical symbol for gold is", |
| "answer": " Au", |
| "generated": " the gold symbol of the sun", |
| "answer_nll": 8.503, |
| "answer_ppl": 4929.396, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The freezing point of water in Celsius is", |
| "answer": " zero", |
| "generated": " the temperature of the water in", |
| "answer_nll": 5.5915, |
| "answer_ppl": 268.131, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The author of Romeo and Juliet is William", |
| "answer": " Shakespeare", |
| "generated": " Shakespeare\u2019s most famous", |
| "answer_nll": 0.6613, |
| "answer_ppl": 1.937, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The Earth revolves around the", |
| "answer": " Sun", |
| "generated": " sun, and the sun is", |
| "answer_nll": 1.2002, |
| "answer_ppl": 3.321, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The plural of mouse is", |
| "answer": " mice", |
| "generated": " the plural of the word mouse", |
| "answer_nll": 11.3772, |
| "answer_ppl": 87309.905, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Photosynthesis occurs in the", |
| "answer": " leaves", |
| "generated": " plant, and the plant produces", |
| "answer_nll": 3.3627, |
| "answer_ppl": 28.868, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The human heart pumps", |
| "answer": " blood", |
| "generated": " blood to the brain, and", |
| "answer_nll": 2.1132, |
| "answer_ppl": 8.275, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The currency used in the United States is the", |
| "answer": " dollar", |
| "generated": " US dollar.\nThe US", |
| "answer_nll": 3.0413, |
| "answer_ppl": 20.932, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The tallest animal in the world is the", |
| "answer": " giraffe", |
| "generated": " elephant.\nThe elephant is", |
| "answer_nll": 2.588, |
| "answer_ppl": 13.303, |
| "hit": false, |
| "token_f1": 0.0 |
| } |
| ], |
| "generation_samples": { |
| "The most important thing about sleep is": "The most important thing about sleep is that it\u2019s a good idea to take some time to relax.\nIt\u2019s also a great idea to set aside time to focus on the day, because if you\u2019re having a bad night, it also helps to put back together the whole night or even go outside to", |
| "Scientists have discovered that": "Scientists have discovered that, in a recent study of about 20,000 New England men in India, the number of people with cancer in the US has doubled, with about one in five of those with a cancer diagnosis. The results of the study are based on studies conducted by the National Cancer Institute (NCI) at", |
| "In the history of mathematics,": "In the history of mathematics, the term has been used to describe the practice of algebraic thinking.\nThe word \u201cmath\u201d has become a term used in the United States, Canada, and the United Kingdom. It refers to the practice of solving problems involving algebra by means of rational algebra.\nThe term \u201c", |
| "Python code for prime number": "Python code for prime number input, for example:\n- I can see the value I want to be as a result of the value from the input. For example, I will write a python with a parameter of 1. And I will make a function from the output (and the\n- I can be just a function", |
| "The best way to learn a new language is": "The best way to learn a new language is to learn about it.\nRead more: How to Make a New Language\nRead more: How to Make a New Language\nThe best way to prepare your child for a new language is to start the conversation in your child\u2019s first language. This language can help learn new words, it" |
| } |
| } |
