| { |
| "hf_repo_id": "aixk/FHxw", |
| "hf_dataset_repo_id": "aixk/chronosense-data-zz", |
| "hf_vocab_dataset": "aixk/mdb6", |
| "local_checkpoint_dir": "./checkpoints_titanmax_zz", |
| "local_dataset_cache_dir": "./cached_dataset_titanmax_zz", |
| "local_tokenizer_cache_dir": "./cached_tokenizer_titanmax_zz", |
| "web_export_dir": "./vlite_export_titanmax_zz", |
| "max_steps": 15000, |
| "batch_size": 8, |
| "accum_steps": 4, |
| "log_interval": 10, |
| "val_interval": 250, |
| "keep_last_n_ckpts": 2, |
| "learning_rate": 0.0006, |
| "min_lr": 1.5e-05, |
| "warmup_steps": 300, |
| "decay_start_ratio": 0.85, |
| "weight_decay": 0.05, |
| "max_seq_len": 512, |
| "vocab_size": 16384, |
| "concept_size": 8192, |
| "hidden_dim": 768, |
| "num_layers": 12, |
| "num_heads": 12, |
| "head_dim": 64, |
| "intermediate_dim": 2048, |
| "chunk_size": 64, |
| "conv_k": 4, |
| "attention_layers": [ |
| 3, |
| 7, |
| 11 |
| ], |
| "label_smoothing": 0.05, |
| "logit_soft_capping": 30.0, |
| "num_folds": 16, |
| "vlite_quant_mode": "bitnet_1.58b", |
| "architectures": [ |
| "FHN_FS9_TitanMax_zz" |
| ], |
| "model_type": "fhn_titanmax_zz_knowledge_engine", |
| "vlite_format": "vlite_bitnet_1.58b_packed", |
| "vlite_layers": { |
| "blocks.0.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.w_mdot.weight.packed", |
| "scale_key": "blocks.0.w_mdot.weight.scale" |
| }, |
| "blocks.0.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.n1_conv.weight.packed", |
| "scale_key": "blocks.0.n1_conv.weight.scale" |
| }, |
| "blocks.0.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.q_proj.weight.packed", |
| "scale_key": "blocks.0.q_proj.weight.scale" |
| }, |
| "blocks.0.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.k_proj.weight.packed", |
| "scale_key": "blocks.0.k_proj.weight.scale" |
| }, |
| "blocks.0.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.v_proj.weight.packed", |
| "scale_key": "blocks.0.v_proj.weight.scale" |
| }, |
| "blocks.0.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.decay_proj.weight.packed", |
| "scale_key": "blocks.0.decay_proj.weight.scale" |
| }, |
| "blocks.0.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.out_proj.weight.packed", |
| "scale_key": "blocks.0.out_proj.weight.scale" |
| }, |
| "blocks.0.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.ffn.w1.weight.packed", |
| "scale_key": "blocks.0.ffn.w1.weight.scale" |
| }, |
| "blocks.0.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.ffn.w2.weight.packed", |
| "scale_key": "blocks.0.ffn.w2.weight.scale" |
| }, |
| "blocks.0.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.0.ffn.w3.weight.packed", |
| "scale_key": "blocks.0.ffn.w3.weight.scale" |
| }, |
| "blocks.1.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.w_mdot.weight.packed", |
| "scale_key": "blocks.1.w_mdot.weight.scale" |
| }, |
| "blocks.1.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.n1_conv.weight.packed", |
| "scale_key": "blocks.1.n1_conv.weight.scale" |
| }, |
| "blocks.1.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.q_proj.weight.packed", |
| "scale_key": "blocks.1.q_proj.weight.scale" |
| }, |
| "blocks.1.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.k_proj.weight.packed", |
| "scale_key": "blocks.1.k_proj.weight.scale" |
| }, |
| "blocks.1.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.v_proj.weight.packed", |
| "scale_key": "blocks.1.v_proj.weight.scale" |
| }, |
| "blocks.1.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.decay_proj.weight.packed", |
| "scale_key": "blocks.1.decay_proj.weight.scale" |
| }, |
| "blocks.1.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.out_proj.weight.packed", |
| "scale_key": "blocks.1.out_proj.weight.scale" |
| }, |
| "blocks.1.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.ffn.w1.weight.packed", |
| "scale_key": "blocks.1.ffn.w1.weight.scale" |
| }, |
| "blocks.1.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.ffn.w2.weight.packed", |
| "scale_key": "blocks.1.ffn.w2.weight.scale" |
| }, |
| "blocks.1.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.1.ffn.w3.weight.packed", |
| "scale_key": "blocks.1.ffn.w3.weight.scale" |
| }, |
| "blocks.2.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.w_mdot.weight.packed", |
| "scale_key": "blocks.2.w_mdot.weight.scale" |
| }, |
| "blocks.2.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.n1_conv.weight.packed", |
| "scale_key": "blocks.2.n1_conv.weight.scale" |
| }, |
| "blocks.2.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.q_proj.weight.packed", |
| "scale_key": "blocks.2.q_proj.weight.scale" |
| }, |
| "blocks.2.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.k_proj.weight.packed", |
| "scale_key": "blocks.2.k_proj.weight.scale" |
| }, |
| "blocks.2.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.v_proj.weight.packed", |
| "scale_key": "blocks.2.v_proj.weight.scale" |
| }, |
| "blocks.2.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.decay_proj.weight.packed", |
| "scale_key": "blocks.2.decay_proj.weight.scale" |
| }, |
| "blocks.2.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.out_proj.weight.packed", |
| "scale_key": "blocks.2.out_proj.weight.scale" |
| }, |
| "blocks.2.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.ffn.w1.weight.packed", |
| "scale_key": "blocks.2.ffn.w1.weight.scale" |
| }, |
| "blocks.2.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.ffn.w2.weight.packed", |
| "scale_key": "blocks.2.ffn.w2.weight.scale" |
| }, |
| "blocks.2.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.2.ffn.w3.weight.packed", |
| "scale_key": "blocks.2.ffn.w3.weight.scale" |
| }, |
| "blocks.3.qkv.weight": { |
| "orig_shape": [ |
| 2304, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.3.qkv.weight.packed", |
| "scale_key": "blocks.3.qkv.weight.scale" |
| }, |
| "blocks.3.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.3.out_proj.weight.packed", |
| "scale_key": "blocks.3.out_proj.weight.scale" |
| }, |
| "blocks.3.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.3.ffn.w1.weight.packed", |
| "scale_key": "blocks.3.ffn.w1.weight.scale" |
| }, |
| "blocks.3.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.3.ffn.w2.weight.packed", |
| "scale_key": "blocks.3.ffn.w2.weight.scale" |
| }, |
| "blocks.3.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.3.ffn.w3.weight.packed", |
| "scale_key": "blocks.3.ffn.w3.weight.scale" |
| }, |
| "blocks.4.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.w_mdot.weight.packed", |
| "scale_key": "blocks.4.w_mdot.weight.scale" |
| }, |
| "blocks.4.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.n1_conv.weight.packed", |
| "scale_key": "blocks.4.n1_conv.weight.scale" |
| }, |
| "blocks.4.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.q_proj.weight.packed", |
| "scale_key": "blocks.4.q_proj.weight.scale" |
| }, |
| "blocks.4.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.k_proj.weight.packed", |
| "scale_key": "blocks.4.k_proj.weight.scale" |
| }, |
| "blocks.4.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.v_proj.weight.packed", |
| "scale_key": "blocks.4.v_proj.weight.scale" |
| }, |
| "blocks.4.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.decay_proj.weight.packed", |
| "scale_key": "blocks.4.decay_proj.weight.scale" |
| }, |
| "blocks.4.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.out_proj.weight.packed", |
| "scale_key": "blocks.4.out_proj.weight.scale" |
| }, |
| "blocks.4.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.ffn.w1.weight.packed", |
| "scale_key": "blocks.4.ffn.w1.weight.scale" |
| }, |
| "blocks.4.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.ffn.w2.weight.packed", |
| "scale_key": "blocks.4.ffn.w2.weight.scale" |
| }, |
| "blocks.4.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.4.ffn.w3.weight.packed", |
| "scale_key": "blocks.4.ffn.w3.weight.scale" |
| }, |
| "blocks.5.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.w_mdot.weight.packed", |
| "scale_key": "blocks.5.w_mdot.weight.scale" |
| }, |
| "blocks.5.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.n1_conv.weight.packed", |
| "scale_key": "blocks.5.n1_conv.weight.scale" |
| }, |
| "blocks.5.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.q_proj.weight.packed", |
| "scale_key": "blocks.5.q_proj.weight.scale" |
| }, |
| "blocks.5.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.k_proj.weight.packed", |
| "scale_key": "blocks.5.k_proj.weight.scale" |
| }, |
| "blocks.5.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.v_proj.weight.packed", |
| "scale_key": "blocks.5.v_proj.weight.scale" |
| }, |
| "blocks.5.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.decay_proj.weight.packed", |
| "scale_key": "blocks.5.decay_proj.weight.scale" |
| }, |
| "blocks.5.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.out_proj.weight.packed", |
| "scale_key": "blocks.5.out_proj.weight.scale" |
| }, |
| "blocks.5.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.ffn.w1.weight.packed", |
| "scale_key": "blocks.5.ffn.w1.weight.scale" |
| }, |
| "blocks.5.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.ffn.w2.weight.packed", |
| "scale_key": "blocks.5.ffn.w2.weight.scale" |
| }, |
| "blocks.5.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.5.ffn.w3.weight.packed", |
| "scale_key": "blocks.5.ffn.w3.weight.scale" |
| }, |
| "blocks.6.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.w_mdot.weight.packed", |
| "scale_key": "blocks.6.w_mdot.weight.scale" |
| }, |
| "blocks.6.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.n1_conv.weight.packed", |
| "scale_key": "blocks.6.n1_conv.weight.scale" |
| }, |
| "blocks.6.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.q_proj.weight.packed", |
| "scale_key": "blocks.6.q_proj.weight.scale" |
| }, |
| "blocks.6.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.k_proj.weight.packed", |
| "scale_key": "blocks.6.k_proj.weight.scale" |
| }, |
| "blocks.6.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.v_proj.weight.packed", |
| "scale_key": "blocks.6.v_proj.weight.scale" |
| }, |
| "blocks.6.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.decay_proj.weight.packed", |
| "scale_key": "blocks.6.decay_proj.weight.scale" |
| }, |
| "blocks.6.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.out_proj.weight.packed", |
| "scale_key": "blocks.6.out_proj.weight.scale" |
| }, |
| "blocks.6.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.ffn.w1.weight.packed", |
| "scale_key": "blocks.6.ffn.w1.weight.scale" |
| }, |
| "blocks.6.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.ffn.w2.weight.packed", |
| "scale_key": "blocks.6.ffn.w2.weight.scale" |
| }, |
| "blocks.6.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.6.ffn.w3.weight.packed", |
| "scale_key": "blocks.6.ffn.w3.weight.scale" |
| }, |
| "blocks.7.qkv.weight": { |
| "orig_shape": [ |
| 2304, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.7.qkv.weight.packed", |
| "scale_key": "blocks.7.qkv.weight.scale" |
| }, |
| "blocks.7.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.7.out_proj.weight.packed", |
| "scale_key": "blocks.7.out_proj.weight.scale" |
| }, |
| "blocks.7.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.7.ffn.w1.weight.packed", |
| "scale_key": "blocks.7.ffn.w1.weight.scale" |
| }, |
| "blocks.7.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.7.ffn.w2.weight.packed", |
| "scale_key": "blocks.7.ffn.w2.weight.scale" |
| }, |
| "blocks.7.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.7.ffn.w3.weight.packed", |
| "scale_key": "blocks.7.ffn.w3.weight.scale" |
| }, |
| "blocks.8.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.w_mdot.weight.packed", |
| "scale_key": "blocks.8.w_mdot.weight.scale" |
| }, |
| "blocks.8.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.n1_conv.weight.packed", |
| "scale_key": "blocks.8.n1_conv.weight.scale" |
| }, |
| "blocks.8.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.q_proj.weight.packed", |
| "scale_key": "blocks.8.q_proj.weight.scale" |
| }, |
| "blocks.8.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.k_proj.weight.packed", |
| "scale_key": "blocks.8.k_proj.weight.scale" |
| }, |
| "blocks.8.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.v_proj.weight.packed", |
| "scale_key": "blocks.8.v_proj.weight.scale" |
| }, |
| "blocks.8.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.decay_proj.weight.packed", |
| "scale_key": "blocks.8.decay_proj.weight.scale" |
| }, |
| "blocks.8.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.out_proj.weight.packed", |
| "scale_key": "blocks.8.out_proj.weight.scale" |
| }, |
| "blocks.8.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.ffn.w1.weight.packed", |
| "scale_key": "blocks.8.ffn.w1.weight.scale" |
| }, |
| "blocks.8.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.ffn.w2.weight.packed", |
| "scale_key": "blocks.8.ffn.w2.weight.scale" |
| }, |
| "blocks.8.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.8.ffn.w3.weight.packed", |
| "scale_key": "blocks.8.ffn.w3.weight.scale" |
| }, |
| "blocks.9.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.w_mdot.weight.packed", |
| "scale_key": "blocks.9.w_mdot.weight.scale" |
| }, |
| "blocks.9.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.n1_conv.weight.packed", |
| "scale_key": "blocks.9.n1_conv.weight.scale" |
| }, |
| "blocks.9.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.q_proj.weight.packed", |
| "scale_key": "blocks.9.q_proj.weight.scale" |
| }, |
| "blocks.9.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.k_proj.weight.packed", |
| "scale_key": "blocks.9.k_proj.weight.scale" |
| }, |
| "blocks.9.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.v_proj.weight.packed", |
| "scale_key": "blocks.9.v_proj.weight.scale" |
| }, |
| "blocks.9.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.decay_proj.weight.packed", |
| "scale_key": "blocks.9.decay_proj.weight.scale" |
| }, |
| "blocks.9.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.out_proj.weight.packed", |
| "scale_key": "blocks.9.out_proj.weight.scale" |
| }, |
| "blocks.9.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.ffn.w1.weight.packed", |
| "scale_key": "blocks.9.ffn.w1.weight.scale" |
| }, |
| "blocks.9.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.ffn.w2.weight.packed", |
| "scale_key": "blocks.9.ffn.w2.weight.scale" |
| }, |
| "blocks.9.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.9.ffn.w3.weight.packed", |
| "scale_key": "blocks.9.ffn.w3.weight.scale" |
| }, |
| "blocks.10.w_mdot.weight": { |
| "orig_shape": [ |
| 1, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.w_mdot.weight.packed", |
| "scale_key": "blocks.10.w_mdot.weight.scale" |
| }, |
| "blocks.10.n1_conv.weight": { |
| "orig_shape": [ |
| 768, |
| 1, |
| 4 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.n1_conv.weight.packed", |
| "scale_key": "blocks.10.n1_conv.weight.scale" |
| }, |
| "blocks.10.q_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.q_proj.weight.packed", |
| "scale_key": "blocks.10.q_proj.weight.scale" |
| }, |
| "blocks.10.k_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.k_proj.weight.packed", |
| "scale_key": "blocks.10.k_proj.weight.scale" |
| }, |
| "blocks.10.v_proj.weight": { |
| "orig_shape": [ |
| 384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.v_proj.weight.packed", |
| "scale_key": "blocks.10.v_proj.weight.scale" |
| }, |
| "blocks.10.decay_proj.weight": { |
| "orig_shape": [ |
| 12, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.decay_proj.weight.packed", |
| "scale_key": "blocks.10.decay_proj.weight.scale" |
| }, |
| "blocks.10.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 384 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.out_proj.weight.packed", |
| "scale_key": "blocks.10.out_proj.weight.scale" |
| }, |
| "blocks.10.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.ffn.w1.weight.packed", |
| "scale_key": "blocks.10.ffn.w1.weight.scale" |
| }, |
| "blocks.10.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.ffn.w2.weight.packed", |
| "scale_key": "blocks.10.ffn.w2.weight.scale" |
| }, |
| "blocks.10.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.10.ffn.w3.weight.packed", |
| "scale_key": "blocks.10.ffn.w3.weight.scale" |
| }, |
| "blocks.11.qkv.weight": { |
| "orig_shape": [ |
| 2304, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.11.qkv.weight.packed", |
| "scale_key": "blocks.11.qkv.weight.scale" |
| }, |
| "blocks.11.out_proj.weight": { |
| "orig_shape": [ |
| 768, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.11.out_proj.weight.packed", |
| "scale_key": "blocks.11.out_proj.weight.scale" |
| }, |
| "blocks.11.ffn.w1.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.11.ffn.w1.weight.packed", |
| "scale_key": "blocks.11.ffn.w1.weight.scale" |
| }, |
| "blocks.11.ffn.w2.weight": { |
| "orig_shape": [ |
| 2048, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.11.ffn.w2.weight.packed", |
| "scale_key": "blocks.11.ffn.w2.weight.scale" |
| }, |
| "blocks.11.ffn.w3.weight": { |
| "orig_shape": [ |
| 768, |
| 2048 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "blocks.11.ffn.w3.weight.packed", |
| "scale_key": "blocks.11.ffn.w3.weight.scale" |
| }, |
| "lm_head.weight": { |
| "orig_shape": [ |
| 16384, |
| 768 |
| ], |
| "quant_mode": "bitnet_1.58b", |
| "packed_key": "lm_head.weight.packed", |
| "scale_key": "lm_head.weight.scale" |
| } |
| }, |
| "torch_dtype": "float16" |
| } |
