{
 "interp_matmul_max_abs_err": 2.384185791015625e-07,
 "torch": "2.14.1+cpu",
 "errors": {},
 "tinystories": {
  "levels": [
   1,
   8,
   16,
   24,
   32,
   40,
   52,
   64,
   76,
   92,
   108,
   128
  ],
  "patched_vs_official_greedy_codes_exact": "4/4",
  "onnx_mb": 573.2,
  "ops": [
   "Add",
   "ArgMax",
   "Cast",
   "Clip",
   "Concat",
   "Constant",
   "ConstantOfShape",
   "Conv",
   "ConvTranspose",
   "CumSum",
   "Div",
   "Equal",
   "Erf",
   "Exp",
   "Expand",
   "Gather",
   "Identity",
   "LayerNormalization",
   "Less",
   "MatMul",
   "Mul",
   "Neg",
   "Pad",
   "Pow",
   "ReduceMean",
   "ReduceSum",
   "Reshape",
   "ScatterND",
   "Shape",
   "Slice",
   "Softmax",
   "Split",
   "Sqrt",
   "Sub",
   "Transpose",
   "Unsqueeze",
   "Where"
  ],
  "example_codes_head": [
   1468,
   2894,
   2139,
   937,
   36,
   2606,
   1043,
   3624,
   280,
   904,
   3735,
   4041,
   1818,
   2780,
   665,
   2459
  ],
  "ort_vs_torch_neargreedy_codes_exact": "4/4",
  "ort_vs_torch_sampled_first_divergence": [
   {
    "pos": 84,
    "level": 5,
    "frac_equal": 0.8097
   },
   {
    "pos": 156,
    "level": 6,
    "frac_equal": 0.9111
   },
   {
    "pos": 10,
    "level": 2,
    "frac_equal": 0.1966
   },
   {
    "pos": 44,
    "level": 3,
    "frac_equal": 0.4306
   }
  ],
  "ort_vs_torch_codes_exact": "0/4",
  "ort_vs_torch_tokens_exact": "0/4",
  "seconds": 125.9
 },
 "sudoku-easy": {
  "levels": [
   1,
   2,
   4,
   6,
   9,
   13,
   18,
   26,
   37,
   50,
   67,
   89
  ],
  "patched_vs_official_exact": "4/4",
  "onnx_mb": 152.5,
  "ops": [
   "Add",
   "ArgMax",
   "Cast",
   "Concat",
   "Constant",
   "ConstantOfShape",
   "Conv",
   "ConvTranspose",
   "Div",
   "Equal",
   "Erf",
   "Expand",
   "Gather",
   "Identity",
   "LayerNormalization",
   "MatMul",
   "Mul",
   "Neg",
   "Pow",
   "ReduceMean",
   "Reshape",
   "ScatterND",
   "Shape",
   "Slice",
   "Softmax",
   "Split",
   "Sqrt",
   "Transpose",
   "Unsqueeze",
   "Where"
  ],
  "example_codes_head": [
   14,
   11,
   8,
   14,
   15,
   6,
   10,
   6,
   11,
   14,
   3,
   14,
   10,
   12,
   12,
   12
  ],
  "ort_vs_torch_codes_exact": "4/4",
  "ort_vs_torch_tokens_exact": "4/4",
  "seconds": 28.5
 },
 "sudoku-medium": {
  "levels": [
   1,
   2,
   4,
   6,
   9,
   13,
   18,
   26,
   37,
   50,
   67,
   89
  ],
  "patched_vs_official_exact": "4/4",
  "onnx_mb": 152.5,
  "ops": [
   "Add",
   "ArgMax",
   "Cast",
   "Concat",
   "Constant",
   "ConstantOfShape",
   "Conv",
   "ConvTranspose",
   "Div",
   "Equal",
   "Erf",
   "Expand",
   "Gather",
   "Identity",
   "LayerNormalization",
   "MatMul",
   "Mul",
   "Neg",
   "Pow",
   "ReduceMean",
   "Reshape",
   "ScatterND",
   "Shape",
   "Slice",
   "Softmax",
   "Split",
   "Sqrt",
   "Transpose",
   "Unsqueeze",
   "Where"
  ],
  "example_codes_head": [
   15,
   7,
   7,
   8,
   3,
   11,
   8,
   14,
   5,
   13,
   11,
   10,
   9,
   4,
   4,
   4
  ],
  "ort_vs_torch_codes_exact": "4/4",
  "ort_vs_torch_tokens_exact": "4/4",
  "seconds": 27.5
 },
 "sudoku-hard": {
  "levels": [
   1,
   2,
   4,
   6,
   9,
   13,
   18,
   26,
   37,
   50,
   67,
   89
  ],
  "patched_vs_official_exact": "4/4",
  "onnx_mb": 152.5,
  "ops": [
   "Add",
   "ArgMax",
   "Cast",
   "Concat",
   "Constant",
   "ConstantOfShape",
   "Conv",
   "ConvTranspose",
   "Div",
   "Equal",
   "Erf",
   "Expand",
   "Gather",
   "Identity",
   "LayerNormalization",
   "MatMul",
   "Mul",
   "Neg",
   "Pow",
   "ReduceMean",
   "Reshape",
   "ScatterND",
   "Shape",
   "Slice",
   "Softmax",
   "Split",
   "Sqrt",
   "Transpose",
   "Unsqueeze",
   "Where"
  ],
  "example_codes_head": [
   5,
   15,
   2,
   0,
   11,
   6,
   9,
   5,
   1,
   6,
   9,
   10,
   8,
   11,
   11,
   11
  ],
  "ort_vs_torch_codes_exact": "4/4",
  "ort_vs_torch_tokens_exact": "4/4",
  "seconds": 27.7
 },
 "lm1b-small": {
  "levels": [
   1,
   8,
   16,
   24,
   32,
   40,
   52,
   64,
   76,
   92,
   108,
   128
  ],
  "patched_vs_official_greedy_codes_exact": "4/4",
  "onnx_mb": 769.9,
  "ops": [
   "Add",
   "Cast",
   "Clip",
   "Concat",
   "Constant",
   "ConstantOfShape",
   "Conv",
   "ConvTranspose",
   "CumSum",
   "Div",
   "Equal",
   "Erf",
   "Expand",
   "Gather",
   "Identity",
   "LayerNormalization",
   "Less",
   "MatMul",
   "Mul",
   "Neg",
   "Pad",
   "Pow",
   "ReduceMean",
   "ReduceSum",
   "Reshape",
   "ScatterND",
   "Shape",
   "Slice",
   "Softmax",
   "Split",
   "Sqrt",
   "TopK",
   "Transpose",
   "Unsqueeze",
   "Where"
  ],
  "example_codes_head": [
   1604,
   3839,
   1844,
   1438,
   5678,
   7102,
   7907,
   3608,
   7304,
   4521,
   1465,
   8029,
   3720,
   5731,
   3820,
   1765
  ],
  "ort_vs_torch_neargreedy_codes_exact": "4/4",
  "ort_vs_torch_neargreedy_topk_values_max_abs_diff": 0.00014257431030273438,
  "ort_vs_torch_sampled_first_divergence": [
   {
    "pos": 197,
    "level": 7,
    "frac_equal": 0.8346
   },
   {
    "pos": 212,
    "level": 7,
    "frac_equal": 0.8658
   },
   {
    "pos": 196,
    "level": 7,
    "frac_equal": 0.7769
   },
   {
    "pos": 166,
    "level": 6,
    "frac_equal": 0.7145
   }
  ],
  "ort_vs_torch_codes_exact": "0/4",
  "ort_vs_torch_tokens_exact": "0/4",
  "ort_vs_torch_topk_values_max_abs_diff": 13.146541595458984,
  "seconds": 134.0
 },
 "lm1b_penalty_topk_agreement": "3834/3846"
}