{
  "status": "ok",
  "meta": {
    "device": "cuda",
    "dtype": "torch.float16",
    "hidden": 2048,
    "hidden_effective": 2048,
    "warmup": 20,
    "iters": 120,
    "bucket_lengths": [
      256
    ],
    "batch_sizes": [
      64
    ],
    "torch_compile": true,
    "compile_mode": "default",
    "compile_backend": "inductor",
    "inductor_gemm_backends": "ATEN",
    "inductor_gemm_backends_applied": true,
    "allow_tf32": true,
    "inference_mode": true,
    "use_inplace_residual": true,
    "pad_hidden_to": 16,
    "use_cuda_graph_arena": false,
    "single_shape_seq_len": 0,
    "skip_batch_scaling": false,
    "single_pass_after_warmup": false
  },
  "global_counters": {
    "compile_calls": 1,
    "graph_build_calls": 1
  },
  "bucket_results": [
    {
      "batch": 64,
      "seq_len": 256,
      "samples_ms": [
        15.6434,
        15.6296,
        15.633,
        15.6318,
        15.5237,
        15.5205,
        15.5252,
        15.5191,
        15.53,
        15.6176,
        15.6381,
        15.6287,
        15.6315,
        15.6302,
        15.6376,
        15.6406,
        15.6402,
        15.6363,
        15.6371,
        15.6342,
        15.6646,
        15.6482,
        15.7729,
        15.8675,
        16.0171,
        16.1274,
        16.1155,
        16.0048,
        15.8991,
        15.8746,
        15.8775,
        21.0208,
        16.2104,
        15.4114,
        15.5104,
        15.666,
        15.7924,
        15.8928,
        16.0031,
        15.996,
        16.0032,
        15.9622,
        15.8778,
        15.8895,
        15.7751,
        15.7471,
        15.7569,
        15.7535,
        15.7522,
        15.7419,
        15.7501,
        15.7502,
        15.7497,
        15.7541,
        15.7584,
        15.7549,
        15.7522,
        15.7505,
        15.7622,
        15.7579,
        15.714,
        15.6278,
        15.6329,
        15.6381,
        15.6318,
        15.627,
        15.6253,
        15.6319,
        15.6206,
        15.6274,
        15.6224,
        15.628,
        15.6385,
        15.6388,
        15.6378,
        15.6301,
        15.6336,
        15.646,
        15.6391,
        15.6381,
        15.64,
        15.6309,
        15.6348,
        19.7269,
        15.8662,
        15.5387,
        15.6366,
        15.8224,
        15.8816,
        15.8835,
        15.8772,
        15.8818,
        15.8788,
        15.8793,
        15.8809,
        15.8777,
        15.8747,
        15.8776,
        15.8752,
        15.8841,
        15.8853,
        15.8811,
        15.8878,
        15.8865,
        15.8866,
        15.883,
        15.8811,
        15.8083,
        15.7584,
        15.7492,
        15.7474,
        15.7463,
        15.7424,
        15.7528,
        15.7457,
        15.7408,
        15.7512,
        15.7506,
        15.7514,
        15.7545
      ],
      "latency_ms": {
        "mean": 15.8256,
        "p50": 15.7502,
        "p95": 16.0048,
        "min": 15.4114,
        "max": 21.0208
      },
      "compile_calls_during_steady_state": 0,
      "graph_build_calls_during_steady_state": 0,
      "memory_pre": {
        "allocated_bytes": 142774272,
        "reserved_bytes": 492830720,
        "peak_allocated_bytes": 478310400,
        "allocation_events": 192,
        "segment_alloc_events": 7
      },
      "memory_post": {
        "allocated_bytes": 142774272,
        "reserved_bytes": 492830720,
        "peak_allocated_bytes": 478310400,
        "allocation_events": 912,
        "segment_alloc_events": 7
      },
      "allocation_event_delta": 720,
      "segment_alloc_event_delta": 0,
      "memory_stable": false
    }
  ],
  "batch_scaling": [
    {
      "batch": 64,
      "seq_len": 256,
      "samples_ms": [
        15.8064,
        15.6612,
        15.6491,
        15.6381,
        15.6352,
        15.635,
        15.6376,
        15.64,
        15.6413,
        15.6337,
        15.6387,
        15.6368,
        15.6383,
        15.6457,
        15.6484,
        15.6459,
        15.7004,
        15.7487,
        15.7645,
        15.7591,
        15.7617,
        15.7638,
        15.7766,
        15.8808,
        15.875,
        15.8826,
        15.8731,
        15.8819,
        15.8728,
        15.8794
      ],
      "latency_ms": {
        "mean": 15.7284,
        "p50": 15.6612,
        "p95": 15.8819,
        "min": 15.6337,
        "max": 15.8826
      },
      "compile_calls_during_steady_state": 0,
      "graph_build_calls_during_steady_state": 0,
      "memory_pre": {
        "allocated_bytes": 142774272,
        "reserved_bytes": 492830720,
        "peak_allocated_bytes": 478310400,
        "allocation_events": 972,
        "segment_alloc_events": 9
      },
      "memory_post": {
        "allocated_bytes": 142774272,
        "reserved_bytes": 492830720,
        "peak_allocated_bytes": 478310400,
        "allocation_events": 1152,
        "segment_alloc_events": 9
      },
      "allocation_event_delta": 180,
      "segment_alloc_event_delta": 0,
      "memory_stable": false
    }
  ]
}