| { |
| "interp_matmul_max_abs_err": 2.384185791015625e-07, |
| "torch": "2.14.1+cpu", |
| "errors": {}, |
| "tinystories": { |
| "levels": [ |
| 1, |
| 8, |
| 16, |
| 24, |
| 32, |
| 40, |
| 52, |
| 64, |
| 76, |
| 92, |
| 108, |
| 128 |
| ], |
| "patched_vs_official_greedy_codes_exact": "4/4", |
| "onnx_mb": 573.2, |
| "ops": [ |
| "Add", |
| "ArgMax", |
| "Cast", |
| "Clip", |
| "Concat", |
| "Constant", |
| "ConstantOfShape", |
| "Conv", |
| "ConvTranspose", |
| "CumSum", |
| "Div", |
| "Equal", |
| "Erf", |
| "Exp", |
| "Expand", |
| "Gather", |
| "Identity", |
| "LayerNormalization", |
| "Less", |
| "MatMul", |
| "Mul", |
| "Neg", |
| "Pad", |
| "Pow", |
| "ReduceMean", |
| "ReduceSum", |
| "Reshape", |
| "ScatterND", |
| "Shape", |
| "Slice", |
| "Softmax", |
| "Split", |
| "Sqrt", |
| "Sub", |
| "Transpose", |
| "Unsqueeze", |
| "Where" |
| ], |
| "example_codes_head": [ |
| 1468, |
| 2894, |
| 2139, |
| 937, |
| 36, |
| 2606, |
| 1043, |
| 3624, |
| 280, |
| 904, |
| 3735, |
| 4041, |
| 1818, |
| 2780, |
| 665, |
| 2459 |
| ], |
| "ort_vs_torch_neargreedy_codes_exact": "4/4", |
| "ort_vs_torch_sampled_first_divergence": [ |
| { |
| "pos": 84, |
| "level": 5, |
| "frac_equal": 0.8097 |
| }, |
| { |
| "pos": 156, |
| "level": 6, |
| "frac_equal": 0.9111 |
| }, |
| { |
| "pos": 10, |
| "level": 2, |
| "frac_equal": 0.1966 |
| }, |
| { |
| "pos": 44, |
| "level": 3, |
| "frac_equal": 0.4306 |
| } |
| ], |
| "ort_vs_torch_codes_exact": "0/4", |
| "ort_vs_torch_tokens_exact": "0/4", |
| "seconds": 125.9 |
| }, |
| "sudoku-easy": { |
| "levels": [ |
| 1, |
| 2, |
| 4, |
| 6, |
| 9, |
| 13, |
| 18, |
| 26, |
| 37, |
| 50, |
| 67, |
| 89 |
| ], |
| "patched_vs_official_exact": "4/4", |
| "onnx_mb": 152.5, |
| "ops": [ |
| "Add", |
| "ArgMax", |
| "Cast", |
| "Concat", |
| "Constant", |
| "ConstantOfShape", |
| "Conv", |
| "ConvTranspose", |
| "Div", |
| "Equal", |
| "Erf", |
| "Expand", |
| "Gather", |
| "Identity", |
| "LayerNormalization", |
| "MatMul", |
| "Mul", |
| "Neg", |
| "Pow", |
| "ReduceMean", |
| "Reshape", |
| "ScatterND", |
| "Shape", |
| "Slice", |
| "Softmax", |
| "Split", |
| "Sqrt", |
| "Transpose", |
| "Unsqueeze", |
| "Where" |
| ], |
| "example_codes_head": [ |
| 14, |
| 11, |
| 8, |
| 14, |
| 15, |
| 6, |
| 10, |
| 6, |
| 11, |
| 14, |
| 3, |
| 14, |
| 10, |
| 12, |
| 12, |
| 12 |
| ], |
| "ort_vs_torch_codes_exact": "4/4", |
| "ort_vs_torch_tokens_exact": "4/4", |
| "seconds": 28.5 |
| }, |
| "sudoku-medium": { |
| "levels": [ |
| 1, |
| 2, |
| 4, |
| 6, |
| 9, |
| 13, |
| 18, |
| 26, |
| 37, |
| 50, |
| 67, |
| 89 |
| ], |
| "patched_vs_official_exact": "4/4", |
| "onnx_mb": 152.5, |
| "ops": [ |
| "Add", |
| "ArgMax", |
| "Cast", |
| "Concat", |
| "Constant", |
| "ConstantOfShape", |
| "Conv", |
| "ConvTranspose", |
| "Div", |
| "Equal", |
| "Erf", |
| "Expand", |
| "Gather", |
| "Identity", |
| "LayerNormalization", |
| "MatMul", |
| "Mul", |
| "Neg", |
| "Pow", |
| "ReduceMean", |
| "Reshape", |
| "ScatterND", |
| "Shape", |
| "Slice", |
| "Softmax", |
| "Split", |
| "Sqrt", |
| "Transpose", |
| "Unsqueeze", |
| "Where" |
| ], |
| "example_codes_head": [ |
| 15, |
| 7, |
| 7, |
| 8, |
| 3, |
| 11, |
| 8, |
| 14, |
| 5, |
| 13, |
| 11, |
| 10, |
| 9, |
| 4, |
| 4, |
| 4 |
| ], |
| "ort_vs_torch_codes_exact": "4/4", |
| "ort_vs_torch_tokens_exact": "4/4", |
| "seconds": 27.5 |
| }, |
| "sudoku-hard": { |
| "levels": [ |
| 1, |
| 2, |
| 4, |
| 6, |
| 9, |
| 13, |
| 18, |
| 26, |
| 37, |
| 50, |
| 67, |
| 89 |
| ], |
| "patched_vs_official_exact": "4/4", |
| "onnx_mb": 152.5, |
| "ops": [ |
| "Add", |
| "ArgMax", |
| "Cast", |
| "Concat", |
| "Constant", |
| "ConstantOfShape", |
| "Conv", |
| "ConvTranspose", |
| "Div", |
| "Equal", |
| "Erf", |
| "Expand", |
| "Gather", |
| "Identity", |
| "LayerNormalization", |
| "MatMul", |
| "Mul", |
| "Neg", |
| "Pow", |
| "ReduceMean", |
| "Reshape", |
| "ScatterND", |
| "Shape", |
| "Slice", |
| "Softmax", |
| "Split", |
| "Sqrt", |
| "Transpose", |
| "Unsqueeze", |
| "Where" |
| ], |
| "example_codes_head": [ |
| 5, |
| 15, |
| 2, |
| 0, |
| 11, |
| 6, |
| 9, |
| 5, |
| 1, |
| 6, |
| 9, |
| 10, |
| 8, |
| 11, |
| 11, |
| 11 |
| ], |
| "ort_vs_torch_codes_exact": "4/4", |
| "ort_vs_torch_tokens_exact": "4/4", |
| "seconds": 27.7 |
| }, |
| "lm1b-small": { |
| "levels": [ |
| 1, |
| 8, |
| 16, |
| 24, |
| 32, |
| 40, |
| 52, |
| 64, |
| 76, |
| 92, |
| 108, |
| 128 |
| ], |
| "patched_vs_official_greedy_codes_exact": "4/4", |
| "onnx_mb": 769.9, |
| "ops": [ |
| "Add", |
| "Cast", |
| "Clip", |
| "Concat", |
| "Constant", |
| "ConstantOfShape", |
| "Conv", |
| "ConvTranspose", |
| "CumSum", |
| "Div", |
| "Equal", |
| "Erf", |
| "Expand", |
| "Gather", |
| "Identity", |
| "LayerNormalization", |
| "Less", |
| "MatMul", |
| "Mul", |
| "Neg", |
| "Pad", |
| "Pow", |
| "ReduceMean", |
| "ReduceSum", |
| "Reshape", |
| "ScatterND", |
| "Shape", |
| "Slice", |
| "Softmax", |
| "Split", |
| "Sqrt", |
| "TopK", |
| "Transpose", |
| "Unsqueeze", |
| "Where" |
| ], |
| "example_codes_head": [ |
| 1604, |
| 3839, |
| 1844, |
| 1438, |
| 5678, |
| 7102, |
| 7907, |
| 3608, |
| 7304, |
| 4521, |
| 1465, |
| 8029, |
| 3720, |
| 5731, |
| 3820, |
| 1765 |
| ], |
| "ort_vs_torch_neargreedy_codes_exact": "4/4", |
| "ort_vs_torch_neargreedy_topk_values_max_abs_diff": 0.00014257431030273438, |
| "ort_vs_torch_sampled_first_divergence": [ |
| { |
| "pos": 197, |
| "level": 7, |
| "frac_equal": 0.8346 |
| }, |
| { |
| "pos": 212, |
| "level": 7, |
| "frac_equal": 0.8658 |
| }, |
| { |
| "pos": 196, |
| "level": 7, |
| "frac_equal": 0.7769 |
| }, |
| { |
| "pos": 166, |
| "level": 6, |
| "frac_equal": 0.7145 |
| } |
| ], |
| "ort_vs_torch_codes_exact": "0/4", |
| "ort_vs_torch_tokens_exact": "0/4", |
| "ort_vs_torch_topk_values_max_abs_diff": 13.146541595458984, |
| "seconds": 134.0 |
| }, |
| "lm1b_penalty_topk_agreement": "3834/3846" |
| } |
