generation_config.json

{
  "_from_model_config": true,
  "bos_token_id": 151643,
  "eos_token_id": 151645,
  "transformers_version": "4.57.1",
  "cache_implementation": "quantized",
  "cache_config": {
    "backend": "hqq",
    "nbits": 8,
    "axis_key": 0,
    "axis_value": 0,
    "q_group_size": 64,
    "residual_length": 128
  }
}