config.json

{
  "language_to_script": {
    "<en>": "<en>"
  },
  "train_size": {
    "en": 2050000
  },
  "val_size": {
    "en": 10000
  },
  "n_head": 8,
  "d_head": 32,
  "d_model": 1024,
  "d_inner": 8192,
  "dropout": 0.1,
  "dropatt": 0.1,
  "pre_lnorm": false,
  "init": "normal",
  "emb_init": "normal",
  "init_range": 0.1,
  "emb_init_range": 0.01,
  "init_std": 0.02,
  "proj_init_std": 0.01,
  "model_config": "[22, (0,), 0, 0]",
  "activation_function": "gelu",
  "num_predictors": 1,
  "use_binomial": false,
  "s_lower_bound": 3,
  "scale_bp": 10,
  "scale_loss2": 0.6,
  "lm_use_tokens": true,
  "lm_tie_weights": false,
  "lm_rms_eps": 1e-06,
  "use_group_attn": false,
  "attn_type": null,
  "boundaries_type": "none",
  "tokenizer_path": "google/byt5-small",
  "fixed_sf": 5,
  "spikes_left": 2,
  "temp": 0.5,
  "prior": null,
  "script_tokens": [
    "<en>",
    "<eot>"
  ],
  "prior_list": [
    0.333
  ],
  "prior_std": [
    0.023
  ],
  "optim": "adam",
  "lr": 0.0003,
  "reset_lr": false,
  "scheduler": "cosine",
  "warmup_step": 2000,
  "clip": 0.25,
  "weight_decay": 0,
  "adam_b1": 0.9,
  "adam_b2": 0.999,
  "adam_eps": 1e-08,
  "task_type": "LM",
  "max_train_steps": 48500,
  "batch_size": 32,
  "seed": 42,
  "seq_len": 4096,
  "gradient_accumulation_steps": 4,
  "num_warmup_steps": 2000,
  "line_by_line": true,
  "num_train_epochs": 1,
  "freeze_pretrained": false,
  "eval_max_steps": 5000,
  "eval_batch_size": 32,
  "shuffle": true,
  "roll": true,
  "fp16": true,
  "boundary_supervision": false,
  "learn_prior": false,
  "use_bytle": false,
  "is_flexitokens": false,
  "use_bicausal_mask": false,
  "utf_type": null,
  "add_eot_embed": false,
  "lm_hidden_dim": 1024,
  "lm_inter_dim": 4096,
  "lm_re_base": 100000,
  "lm_max_position_embeddings": 4096,
  "lm_n_heads": 8,
  "lm_n_kv_heads": 8,
  "lm_dropout": 0.1,
  "lm_attn_scaling": 1.0,
  "lm_max_length": 4096,
  "prior_one": 0.5,
  "prior_two": 0.2,
  "eval_interval": 5000,
  "eval_tgt_len": 4096,
  "eval_total_len": 4096,
  "id_to_script": {
    "259": "<en>",
    "260": "<eot>"
  },
  "all_script_ids_dict": {
    "<en>": [
      0.333,
      0.023
    ]
  },
  "lm_vocab_size": 261,
  "n_token": 261,
  "ignore_index": -100,
  "mtp_layers": 0
}