config.json

{
  "architectures": [
    "SlideOnModel"
  ],
  "auto_map": {
    "AutoConfig": "modeling_slideon.SlideOnConfig",
    "AutoModel": "modeling_slideon.SlideOnModel"
  },
  "backbone_config": {
    "architectures": [
      "ModernBertModel"
    ],
    "attention_bias": false,
    "attention_dropout": 0.0,
    "bos_token_id": 50281,
    "classifier_activation": "gelu",
    "classifier_bias": false,
    "classifier_dropout": 0.0,
    "classifier_pooling": "mean",
    "cls_token_id": 50281,
    "decoder_bias": true,
    "deterministic_flash_attn": false,
    "dtype": "float32",
    "embedding_dropout": 0.0,
    "eos_token_id": 50282,
    "global_attn_every_n_layers": 3,
    "gradient_checkpointing": false,
    "hidden_activation": "gelu",
    "hidden_size": 768,
    "initializer_cutoff_factor": 2.0,
    "initializer_range": 0.02,
    "intermediate_size": 1152,
    "layer_norm_eps": 1e-05,
    "layer_types": [
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention",
      "sliding_attention",
      "sliding_attention",
      "full_attention"
    ],
    "local_attention": 128,
    "max_position_embeddings": 8192,
    "mlp_bias": false,
    "mlp_dropout": 0.0,
    "model_type": "modernbert",
    "norm_bias": false,
    "norm_eps": 1e-05,
    "num_attention_heads": 12,
    "num_hidden_layers": 22,
    "pad_token_id": 50283,
    "position_embedding_type": "absolute",
    "repad_logits_with_grad": false,
    "rope_parameters": {
      "full_attention": {
        "rope_theta": 160000.0,
        "rope_type": "default"
      },
      "sliding_attention": {
        "rope_theta": 10000.0,
        "rope_type": "default"
      }
    },
    "sep_token_id": 50282,
    "sparse_pred_ignore_index": -100,
    "sparse_prediction": false,
    "tie_word_embeddings": true,
    "vocab_size": 50368
  },
  "base_model": "lightonai/DenseOn",
  "base_model_revision": "cb9947ebccb33862d24e3c7ca2edb25e51acd887",
  "checkpoint_type": "retrieval",
  "head_config": {
    "d_lift": 0,
    "d_model": 768,
    "has_decoder": false,
    "n_latents": 32768
  },
  "inference": {
    "document_length": 8192,
    "document_top_k": 0,
    "head_chunk_size": 2048,
    "pooling": "max_relu",
    "prompts": {
      "document": "document: ",
      "query": "query: "
    },
    "query_length": 8192,
    "query_top_k": 0,
    "score": "dot",
    "token_k": 32
  },
  "model_type": "slideon",
  "projection_configs": [],
  "sae_path": "lightonai/SlideOn-SAE",
  "skip_token_ids": [
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    27,
    28,
    29,
    30,
    31,
    32,
    33,
    60,
    61,
    62,
    63,
    64,
    65,
    92,
    93,
    94,
    95
  ],
  "tokenizer_settings": {
    "document_prefix": "",
    "query_prefix": "",
    "skiplist_words": [
      "!",
      "\"",
      "#",
      "$",
      "%",
      "&",
      "'",
      "(",
      ")",
      "*",
      "+",
      ",",
      "-",
      ".",
      "/",
      ":",
      ";",
      "<",
      "=",
      ">",
      "?",
      "@",
      "[",
      "\\",
      "]",
      "^",
      "_",
      "`",
      "{",
      "|",
      "}",
      "~"
    ]
  },
  "transformers_version": "5.3.0"
}