Hugging Face's logo

meihant
/
AnonymousModel-mht

Text Generation
Transformers
Safetensors
English
qwen3_5
pretraining
Model card Files
xet
 Community
meihant commited on
Commit
1498d46
·
verified ·
1 Parent(s): f103bec

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +20 -0
  2. Keen-qwen4b-20b/checkpoint-11202/model.safetensors +3 -0
  3. Keen-qwen4b-20b/checkpoint-11202/tokenizer.json +3 -0
  4. Keen-qwen4b-20b/checkpoint-13449/model.safetensors +3 -0
  5. Keen-qwen4b-20b/checkpoint-13449/tokenizer.json +3 -0
  6. Keen-qwen4b-20b/checkpoint-15698/model.safetensors +3 -0
  7. Keen-qwen4b-20b/checkpoint-15698/tokenizer.json +3 -0
  8. Keen-qwen4b-20b/checkpoint-17953/model.safetensors +3 -0
  9. Keen-qwen4b-20b/checkpoint-17953/tokenizer.json +3 -0
  10. Keen-qwen4b-20b/checkpoint-20218/model.safetensors +3 -0
  11. Keen-qwen4b-20b/checkpoint-20218/tokenizer.json +3 -0
  12. Keen-qwen4b-20b/checkpoint-22478/model.safetensors +3 -0
  13. Keen-qwen4b-20b/checkpoint-22478/tokenizer.json +3 -0
  14. Keen-qwen4b-20b/checkpoint-2253/model.safetensors +3 -0
  15. Keen-qwen4b-20b/checkpoint-2253/tokenizer.json +3 -0
  16. Keen-qwen4b-20b/checkpoint-4453/model.safetensors +3 -0
  17. Keen-qwen4b-20b/checkpoint-4453/tokenizer.json +3 -0
  18. Keen-qwen4b-20b/checkpoint-6701/model.safetensors +3 -0
  19. Keen-qwen4b-20b/checkpoint-6701/tokenizer.json +3 -0
  20. Keen-qwen4b-20b/checkpoint-8956/model.safetensors +3 -0
  21. Keen-qwen4b-20b/checkpoint-8956/tokenizer.json +3 -0
  22. Vanilla-qwen4b-20b/checkpoint-11452/README.md +40 -0
  23. Vanilla-qwen4b-20b/checkpoint-11452/config.json +83 -0
  24. Vanilla-qwen4b-20b/checkpoint-11452/generation_config.json +9 -0
  25. Vanilla-qwen4b-20b/checkpoint-11452/model.safetensors +3 -0
  26. Vanilla-qwen4b-20b/checkpoint-11452/tokenizer.json +3 -0
  27. Vanilla-qwen4b-20b/checkpoint-11452/tokenizer_config.json +47 -0
  28. Vanilla-qwen4b-20b/checkpoint-13362/README.md +40 -0
  29. Vanilla-qwen4b-20b/checkpoint-13362/config.json +83 -0
  30. Vanilla-qwen4b-20b/checkpoint-13362/generation_config.json +9 -0
  31. Vanilla-qwen4b-20b/checkpoint-13362/model.safetensors +3 -0
  32. Vanilla-qwen4b-20b/checkpoint-13362/tokenizer.json +3 -0
  33. Vanilla-qwen4b-20b/checkpoint-13362/tokenizer_config.json +47 -0
  34. Vanilla-qwen4b-20b/checkpoint-15273/README.md +40 -0
  35. Vanilla-qwen4b-20b/checkpoint-15273/config.json +83 -0
  36. Vanilla-qwen4b-20b/checkpoint-15273/generation_config.json +9 -0
  37. Vanilla-qwen4b-20b/checkpoint-15273/model.safetensors +3 -0
  38. Vanilla-qwen4b-20b/checkpoint-15273/tokenizer.json +3 -0
  39. Vanilla-qwen4b-20b/checkpoint-15273/tokenizer_config.json +47 -0
  40. Vanilla-qwen4b-20b/checkpoint-17188/README.md +40 -0
  41. Vanilla-qwen4b-20b/checkpoint-17188/config.json +83 -0
  42. Vanilla-qwen4b-20b/checkpoint-17188/generation_config.json +9 -0
  43. Vanilla-qwen4b-20b/checkpoint-17188/model.safetensors +3 -0
  44. Vanilla-qwen4b-20b/checkpoint-17188/tokenizer.json +3 -0
  45. Vanilla-qwen4b-20b/checkpoint-17188/tokenizer_config.json +47 -0
  46. Vanilla-qwen4b-20b/checkpoint-1910/config.json +83 -0
  47. Vanilla-qwen4b-20b/checkpoint-1910/generation_config.json +9 -0
  48. Vanilla-qwen4b-20b/checkpoint-1910/model.safetensors +3 -0
  49. Vanilla-qwen4b-20b/checkpoint-1910/tokenizer.json +3 -0
  50. Vanilla-qwen4b-20b/checkpoint-1910/tokenizer_config.json +47 -0
.gitattributes CHANGED
@@ -33,3 +33,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ Keen-qwen4b-20b/checkpoint-4453/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ Keen-qwen4b-20b/checkpoint-6701/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ Keen-qwen4b-20b/checkpoint-17953/tokenizer.json filter=lfs diff=lfs merge=lfs -text
39
+ Keen-qwen4b-20b/checkpoint-15698/tokenizer.json filter=lfs diff=lfs merge=lfs -text
40
+ Keen-qwen4b-20b/checkpoint-2253/tokenizer.json filter=lfs diff=lfs merge=lfs -text
41
+ Keen-qwen4b-20b/checkpoint-8956/tokenizer.json filter=lfs diff=lfs merge=lfs -text
42
+ Keen-qwen4b-20b/checkpoint-11202/tokenizer.json filter=lfs diff=lfs merge=lfs -text
43
+ Keen-qwen4b-20b/checkpoint-13449/tokenizer.json filter=lfs diff=lfs merge=lfs -text
44
+ Keen-qwen4b-20b/checkpoint-20218/tokenizer.json filter=lfs diff=lfs merge=lfs -text
45
+ Keen-qwen4b-20b/checkpoint-22478/tokenizer.json filter=lfs diff=lfs merge=lfs -text
46
+ Vanilla-qwen4b-20b/checkpoint-9543/tokenizer.json filter=lfs diff=lfs merge=lfs -text
47
+ Vanilla-qwen4b-20b/checkpoint-1910/tokenizer.json filter=lfs diff=lfs merge=lfs -text
48
+ Vanilla-qwen4b-20b/checkpoint-7635/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
+ Vanilla-qwen4b-20b/checkpoint-11452/tokenizer.json filter=lfs diff=lfs merge=lfs -text
50
+ Vanilla-qwen4b-20b/checkpoint-19106/tokenizer.json filter=lfs diff=lfs merge=lfs -text
51
+ Vanilla-qwen4b-20b/checkpoint-3818/tokenizer.json filter=lfs diff=lfs merge=lfs -text
52
+ Vanilla-qwen4b-20b/checkpoint-17188/tokenizer.json filter=lfs diff=lfs merge=lfs -text
53
+ Vanilla-qwen4b-20b/checkpoint-5726/tokenizer.json filter=lfs diff=lfs merge=lfs -text
54
+ Vanilla-qwen4b-20b/checkpoint-13362/tokenizer.json filter=lfs diff=lfs merge=lfs -text
55
+ Vanilla-qwen4b-20b/checkpoint-15273/tokenizer.json filter=lfs diff=lfs merge=lfs -text
Keen-qwen4b-20b/checkpoint-11202/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e5c178a11a7f4d1f2c4486a1e21a47acfba0998e2484122c6dbe2e9f61f9101
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-11202/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-13449/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fe83ee3b190b8772bef64da62b031b09516d3666f23deaa2fb32b10416050a6
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-13449/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-15698/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e19c76755aa3b2ce6c0cdf0d170f8d35df339a20e07fb25c9d82cb1d7d22786c
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-15698/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-17953/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:701a90729ccc31bada5fc4dff6e90a8ac7b90afbbb85f8f80816d693e65dc296
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-17953/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-20218/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afdc1e1f396a76fe3cabfac49d643dda0e1019743e5350f8a4e67cd16e668866
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-20218/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-22478/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b869467d5db824e6df4a995bfaca0ae74a7b8e43899593b877b774bcc3861811
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-22478/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-2253/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7816a1cae0b08b1e0901c378b916df2769b9b7fa7ebfac63e98854c33b656a4
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-2253/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-4453/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bcfa8a9a42d9b1362d339b13e364fe04ae52c9460f58a715901f3cac64c1a44
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-4453/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-6701/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94e659545b1ba9eecb5c9a2f03206e893bd544f9b4388e591b4316cabf7f1120
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-6701/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Keen-qwen4b-20b/checkpoint-8956/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d54e7eb42dffdfc89613913ba3a6c7bfcf0e9eb66ad193c978608eb652efc4a
3
+ size 8411554440
Keen-qwen4b-20b/checkpoint-8956/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-11452/README.md ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Vanilla-qwen4b-20b: checkpoint-11452
2
+
3
+ Text-only Qwen3.5-4B architecture, trained from random initialization.
4
+ This is a base pretraining checkpoint, not an instruction-tuned chat model.
5
+ Training used 4,096-token packs with document-isolated attention.
6
+ The larger positional limit in the architecture configuration does not imply
7
+ validated long-context capability beyond the training context.
8
+
9
+ - Optimizer step: 11452
10
+ - Cumulative body tokens: 12,000,591,962
11
+ - Parameters: 4,205,751,296
12
+ - Precision: BF16
13
+ - Special document-end token: `<|endoftext|>`
14
+
15
+ Vanilla uses ordinary passage continuation without a facts prefix.
16
+
17
+ Use Transformers 5.6.2 (the training/evaluation version) with its native
18
+ `Qwen3_5ForCausalLM` implementation. Install PyTorch and Accelerate for GPU loading.
19
+ No remote custom model code is required. Use plain text prompts rather than a
20
+ chat template. Inference caching is enabled in the released configuration;
21
+ weights and tokenizer vocabulary are unchanged.
22
+
23
+ ```python
24
+ import torch
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+
27
+ repo = "meihant/AnonymousModel-mht"
28
+ subfolder = "Vanilla-qwen4b-20b/checkpoint-11452"
29
+ tokenizer = AutoTokenizer.from_pretrained(repo, subfolder=subfolder)
30
+ model = AutoModelForCausalLM.from_pretrained(
31
+ repo, subfolder=subfolder, dtype=torch.bfloat16,
32
+ device_map="auto", attn_implementation="eager",
33
+ ).eval()
34
+ inputs = tokenizer("Paris", return_tensors="pt").to(model.device)
35
+ with torch.inference_mode():
36
+ output = model.generate(**inputs, max_new_tokens=64, do_sample=False,
37
+ use_cache=True, pad_token_id=tokenizer.eos_token_id)
38
+ print(tokenizer.decode(output[0], skip_special_tokens=False))
39
+ ```
40
+
Vanilla-qwen4b-20b/checkpoint-11452/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 9216,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention"
50
+ ],
51
+ "linear_conv_kernel_dim": 4,
52
+ "linear_key_head_dim": 128,
53
+ "linear_num_key_heads": 16,
54
+ "linear_num_value_heads": 32,
55
+ "linear_value_head_dim": 128,
56
+ "mamba_ssm_dtype": "float32",
57
+ "max_position_embeddings": 262144,
58
+ "mlp_only_layers": [],
59
+ "model_type": "qwen3_5_text",
60
+ "mtp_num_hidden_layers": 1,
61
+ "mtp_use_dedicated_embeddings": false,
62
+ "num_attention_heads": 16,
63
+ "num_hidden_layers": 32,
64
+ "num_key_value_heads": 4,
65
+ "pad_token_id": 248044,
66
+ "partial_rotary_factor": 0.25,
67
+ "rms_norm_eps": 1e-06,
68
+ "rope_parameters": {
69
+ "mrope_interleaved": true,
70
+ "mrope_section": [
71
+ 11,
72
+ 11,
73
+ 10
74
+ ],
75
+ "partial_rotary_factor": 0.25,
76
+ "rope_theta": 10000000,
77
+ "rope_type": "default"
78
+ },
79
+ "tie_word_embeddings": true,
80
+ "transformers_version": "5.6.2",
81
+ "use_cache": true,
82
+ "vocab_size": 248320
83
+ }
Vanilla-qwen4b-20b/checkpoint-11452/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 248044,
7
+ "transformers_version": "5.6.2",
8
+ "use_cache": true
9
+ }
Vanilla-qwen4b-20b/checkpoint-11452/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7009776caa28cd43561b57da0d4e9f3538296df42ffa2807f86a729172c20df
3
+ size 8411554440
Vanilla-qwen4b-20b/checkpoint-11452/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-11452/tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "extra_special_tokens": [
12
+ "<|im_start|>",
13
+ "<|im_end|>",
14
+ "<|object_ref_start|>",
15
+ "<|object_ref_end|>",
16
+ "<|box_start|>",
17
+ "<|box_end|>",
18
+ "<|quad_start|>",
19
+ "<|quad_end|>",
20
+ "<|vision_start|>",
21
+ "<|vision_end|>",
22
+ "<|vision_pad|>",
23
+ "<|image_pad|>",
24
+ "<|video_pad|>",
25
+ "<fact>",
26
+ "</fact>"
27
+ ],
28
+ "image_token": "<|image_pad|>",
29
+ "model_max_length": 32768,
30
+ "model_specific_special_tokens": {
31
+ "audio_bos_token": "<|audio_start|>",
32
+ "audio_eos_token": "<|audio_end|>",
33
+ "audio_token": "<|audio_pad|>",
34
+ "image_token": "<|image_pad|>",
35
+ "video_token": "<|video_pad|>",
36
+ "vision_bos_token": "<|vision_start|>",
37
+ "vision_eos_token": "<|vision_end|>"
38
+ },
39
+ "pad_token": "<|endoftext|>",
40
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null,
44
+ "video_token": "<|video_pad|>",
45
+ "vision_bos_token": "<|vision_start|>",
46
+ "vision_eos_token": "<|vision_end|>"
47
+ }
Vanilla-qwen4b-20b/checkpoint-13362/README.md ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Vanilla-qwen4b-20b: checkpoint-13362
2
+
3
+ Text-only Qwen3.5-4B architecture, trained from random initialization.
4
+ This is a base pretraining checkpoint, not an instruction-tuned chat model.
5
+ Training used 4,096-token packs with document-isolated attention.
6
+ The larger positional limit in the architecture configuration does not imply
7
+ validated long-context capability beyond the training context.
8
+
9
+ - Optimizer step: 13362
10
+ - Cumulative body tokens: 14,000,166,403
11
+ - Parameters: 4,205,751,296
12
+ - Precision: BF16
13
+ - Special document-end token: `<|endoftext|>`
14
+
15
+ Vanilla uses ordinary passage continuation without a facts prefix.
16
+
17
+ Use Transformers 5.6.2 (the training/evaluation version) with its native
18
+ `Qwen3_5ForCausalLM` implementation. Install PyTorch and Accelerate for GPU loading.
19
+ No remote custom model code is required. Use plain text prompts rather than a
20
+ chat template. Inference caching is enabled in the released configuration;
21
+ weights and tokenizer vocabulary are unchanged.
22
+
23
+ ```python
24
+ import torch
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+
27
+ repo = "meihant/AnonymousModel-mht"
28
+ subfolder = "Vanilla-qwen4b-20b/checkpoint-13362"
29
+ tokenizer = AutoTokenizer.from_pretrained(repo, subfolder=subfolder)
30
+ model = AutoModelForCausalLM.from_pretrained(
31
+ repo, subfolder=subfolder, dtype=torch.bfloat16,
32
+ device_map="auto", attn_implementation="eager",
33
+ ).eval()
34
+ inputs = tokenizer("Paris", return_tensors="pt").to(model.device)
35
+ with torch.inference_mode():
36
+ output = model.generate(**inputs, max_new_tokens=64, do_sample=False,
37
+ use_cache=True, pad_token_id=tokenizer.eos_token_id)
38
+ print(tokenizer.decode(output[0], skip_special_tokens=False))
39
+ ```
40
+
Vanilla-qwen4b-20b/checkpoint-13362/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 9216,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention"
50
+ ],
51
+ "linear_conv_kernel_dim": 4,
52
+ "linear_key_head_dim": 128,
53
+ "linear_num_key_heads": 16,
54
+ "linear_num_value_heads": 32,
55
+ "linear_value_head_dim": 128,
56
+ "mamba_ssm_dtype": "float32",
57
+ "max_position_embeddings": 262144,
58
+ "mlp_only_layers": [],
59
+ "model_type": "qwen3_5_text",
60
+ "mtp_num_hidden_layers": 1,
61
+ "mtp_use_dedicated_embeddings": false,
62
+ "num_attention_heads": 16,
63
+ "num_hidden_layers": 32,
64
+ "num_key_value_heads": 4,
65
+ "pad_token_id": 248044,
66
+ "partial_rotary_factor": 0.25,
67
+ "rms_norm_eps": 1e-06,
68
+ "rope_parameters": {
69
+ "mrope_interleaved": true,
70
+ "mrope_section": [
71
+ 11,
72
+ 11,
73
+ 10
74
+ ],
75
+ "partial_rotary_factor": 0.25,
76
+ "rope_theta": 10000000,
77
+ "rope_type": "default"
78
+ },
79
+ "tie_word_embeddings": true,
80
+ "transformers_version": "5.6.2",
81
+ "use_cache": true,
82
+ "vocab_size": 248320
83
+ }
Vanilla-qwen4b-20b/checkpoint-13362/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 248044,
7
+ "transformers_version": "5.6.2",
8
+ "use_cache": true
9
+ }
Vanilla-qwen4b-20b/checkpoint-13362/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa377d8aebe815164abafc2dd8a85ccc3e44fadc7552e4cb20014167ca09ee74
3
+ size 8411554440
Vanilla-qwen4b-20b/checkpoint-13362/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-13362/tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "extra_special_tokens": [
12
+ "<|im_start|>",
13
+ "<|im_end|>",
14
+ "<|object_ref_start|>",
15
+ "<|object_ref_end|>",
16
+ "<|box_start|>",
17
+ "<|box_end|>",
18
+ "<|quad_start|>",
19
+ "<|quad_end|>",
20
+ "<|vision_start|>",
21
+ "<|vision_end|>",
22
+ "<|vision_pad|>",
23
+ "<|image_pad|>",
24
+ "<|video_pad|>",
25
+ "<fact>",
26
+ "</fact>"
27
+ ],
28
+ "image_token": "<|image_pad|>",
29
+ "model_max_length": 32768,
30
+ "model_specific_special_tokens": {
31
+ "audio_bos_token": "<|audio_start|>",
32
+ "audio_eos_token": "<|audio_end|>",
33
+ "audio_token": "<|audio_pad|>",
34
+ "image_token": "<|image_pad|>",
35
+ "video_token": "<|video_pad|>",
36
+ "vision_bos_token": "<|vision_start|>",
37
+ "vision_eos_token": "<|vision_end|>"
38
+ },
39
+ "pad_token": "<|endoftext|>",
40
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null,
44
+ "video_token": "<|video_pad|>",
45
+ "vision_bos_token": "<|vision_start|>",
46
+ "vision_eos_token": "<|vision_end|>"
47
+ }
Vanilla-qwen4b-20b/checkpoint-15273/README.md ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Vanilla-qwen4b-20b: checkpoint-15273
2
+
3
+ Text-only Qwen3.5-4B architecture, trained from random initialization.
4
+ This is a base pretraining checkpoint, not an instruction-tuned chat model.
5
+ Training used 4,096-token packs with document-isolated attention.
6
+ The larger positional limit in the architecture configuration does not imply
7
+ validated long-context capability beyond the training context.
8
+
9
+ - Optimizer step: 15273
10
+ - Cumulative body tokens: 16,000,320,768
11
+ - Parameters: 4,205,751,296
12
+ - Precision: BF16
13
+ - Special document-end token: `<|endoftext|>`
14
+
15
+ Vanilla uses ordinary passage continuation without a facts prefix.
16
+
17
+ Use Transformers 5.6.2 (the training/evaluation version) with its native
18
+ `Qwen3_5ForCausalLM` implementation. Install PyTorch and Accelerate for GPU loading.
19
+ No remote custom model code is required. Use plain text prompts rather than a
20
+ chat template. Inference caching is enabled in the released configuration;
21
+ weights and tokenizer vocabulary are unchanged.
22
+
23
+ ```python
24
+ import torch
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+
27
+ repo = "meihant/AnonymousModel-mht"
28
+ subfolder = "Vanilla-qwen4b-20b/checkpoint-15273"
29
+ tokenizer = AutoTokenizer.from_pretrained(repo, subfolder=subfolder)
30
+ model = AutoModelForCausalLM.from_pretrained(
31
+ repo, subfolder=subfolder, dtype=torch.bfloat16,
32
+ device_map="auto", attn_implementation="eager",
33
+ ).eval()
34
+ inputs = tokenizer("Paris", return_tensors="pt").to(model.device)
35
+ with torch.inference_mode():
36
+ output = model.generate(**inputs, max_new_tokens=64, do_sample=False,
37
+ use_cache=True, pad_token_id=tokenizer.eos_token_id)
38
+ print(tokenizer.decode(output[0], skip_special_tokens=False))
39
+ ```
40
+
Vanilla-qwen4b-20b/checkpoint-15273/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 9216,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention"
50
+ ],
51
+ "linear_conv_kernel_dim": 4,
52
+ "linear_key_head_dim": 128,
53
+ "linear_num_key_heads": 16,
54
+ "linear_num_value_heads": 32,
55
+ "linear_value_head_dim": 128,
56
+ "mamba_ssm_dtype": "float32",
57
+ "max_position_embeddings": 262144,
58
+ "mlp_only_layers": [],
59
+ "model_type": "qwen3_5_text",
60
+ "mtp_num_hidden_layers": 1,
61
+ "mtp_use_dedicated_embeddings": false,
62
+ "num_attention_heads": 16,
63
+ "num_hidden_layers": 32,
64
+ "num_key_value_heads": 4,
65
+ "pad_token_id": 248044,
66
+ "partial_rotary_factor": 0.25,
67
+ "rms_norm_eps": 1e-06,
68
+ "rope_parameters": {
69
+ "mrope_interleaved": true,
70
+ "mrope_section": [
71
+ 11,
72
+ 11,
73
+ 10
74
+ ],
75
+ "partial_rotary_factor": 0.25,
76
+ "rope_theta": 10000000,
77
+ "rope_type": "default"
78
+ },
79
+ "tie_word_embeddings": true,
80
+ "transformers_version": "5.6.2",
81
+ "use_cache": true,
82
+ "vocab_size": 248320
83
+ }
Vanilla-qwen4b-20b/checkpoint-15273/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 248044,
7
+ "transformers_version": "5.6.2",
8
+ "use_cache": true
9
+ }
Vanilla-qwen4b-20b/checkpoint-15273/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9233f915f89cd02a49cbd8690c5de7625f483dcc697cc2a451137522613e464a
3
+ size 8411554440
Vanilla-qwen4b-20b/checkpoint-15273/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-15273/tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "extra_special_tokens": [
12
+ "<|im_start|>",
13
+ "<|im_end|>",
14
+ "<|object_ref_start|>",
15
+ "<|object_ref_end|>",
16
+ "<|box_start|>",
17
+ "<|box_end|>",
18
+ "<|quad_start|>",
19
+ "<|quad_end|>",
20
+ "<|vision_start|>",
21
+ "<|vision_end|>",
22
+ "<|vision_pad|>",
23
+ "<|image_pad|>",
24
+ "<|video_pad|>",
25
+ "<fact>",
26
+ "</fact>"
27
+ ],
28
+ "image_token": "<|image_pad|>",
29
+ "model_max_length": 32768,
30
+ "model_specific_special_tokens": {
31
+ "audio_bos_token": "<|audio_start|>",
32
+ "audio_eos_token": "<|audio_end|>",
33
+ "audio_token": "<|audio_pad|>",
34
+ "image_token": "<|image_pad|>",
35
+ "video_token": "<|video_pad|>",
36
+ "vision_bos_token": "<|vision_start|>",
37
+ "vision_eos_token": "<|vision_end|>"
38
+ },
39
+ "pad_token": "<|endoftext|>",
40
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null,
44
+ "video_token": "<|video_pad|>",
45
+ "vision_bos_token": "<|vision_start|>",
46
+ "vision_eos_token": "<|vision_end|>"
47
+ }
Vanilla-qwen4b-20b/checkpoint-17188/README.md ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Vanilla-qwen4b-20b: checkpoint-17188
2
+
3
+ Text-only Qwen3.5-4B architecture, trained from random initialization.
4
+ This is a base pretraining checkpoint, not an instruction-tuned chat model.
5
+ Training used 4,096-token packs with document-isolated attention.
6
+ The larger positional limit in the architecture configuration does not imply
7
+ validated long-context capability beyond the training context.
8
+
9
+ - Optimizer step: 17188
10
+ - Cumulative body tokens: 18,000,257,314
11
+ - Parameters: 4,205,751,296
12
+ - Precision: BF16
13
+ - Special document-end token: `<|endoftext|>`
14
+
15
+ Vanilla uses ordinary passage continuation without a facts prefix.
16
+
17
+ Use Transformers 5.6.2 (the training/evaluation version) with its native
18
+ `Qwen3_5ForCausalLM` implementation. Install PyTorch and Accelerate for GPU loading.
19
+ No remote custom model code is required. Use plain text prompts rather than a
20
+ chat template. Inference caching is enabled in the released configuration;
21
+ weights and tokenizer vocabulary are unchanged.
22
+
23
+ ```python
24
+ import torch
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+
27
+ repo = "meihant/AnonymousModel-mht"
28
+ subfolder = "Vanilla-qwen4b-20b/checkpoint-17188"
29
+ tokenizer = AutoTokenizer.from_pretrained(repo, subfolder=subfolder)
30
+ model = AutoModelForCausalLM.from_pretrained(
31
+ repo, subfolder=subfolder, dtype=torch.bfloat16,
32
+ device_map="auto", attn_implementation="eager",
33
+ ).eval()
34
+ inputs = tokenizer("Paris", return_tensors="pt").to(model.device)
35
+ with torch.inference_mode():
36
+ output = model.generate(**inputs, max_new_tokens=64, do_sample=False,
37
+ use_cache=True, pad_token_id=tokenizer.eos_token_id)
38
+ print(tokenizer.decode(output[0], skip_special_tokens=False))
39
+ ```
40
+
Vanilla-qwen4b-20b/checkpoint-17188/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 9216,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention"
50
+ ],
51
+ "linear_conv_kernel_dim": 4,
52
+ "linear_key_head_dim": 128,
53
+ "linear_num_key_heads": 16,
54
+ "linear_num_value_heads": 32,
55
+ "linear_value_head_dim": 128,
56
+ "mamba_ssm_dtype": "float32",
57
+ "max_position_embeddings": 262144,
58
+ "mlp_only_layers": [],
59
+ "model_type": "qwen3_5_text",
60
+ "mtp_num_hidden_layers": 1,
61
+ "mtp_use_dedicated_embeddings": false,
62
+ "num_attention_heads": 16,
63
+ "num_hidden_layers": 32,
64
+ "num_key_value_heads": 4,
65
+ "pad_token_id": 248044,
66
+ "partial_rotary_factor": 0.25,
67
+ "rms_norm_eps": 1e-06,
68
+ "rope_parameters": {
69
+ "mrope_interleaved": true,
70
+ "mrope_section": [
71
+ 11,
72
+ 11,
73
+ 10
74
+ ],
75
+ "partial_rotary_factor": 0.25,
76
+ "rope_theta": 10000000,
77
+ "rope_type": "default"
78
+ },
79
+ "tie_word_embeddings": true,
80
+ "transformers_version": "5.6.2",
81
+ "use_cache": true,
82
+ "vocab_size": 248320
83
+ }
Vanilla-qwen4b-20b/checkpoint-17188/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 248044,
7
+ "transformers_version": "5.6.2",
8
+ "use_cache": true
9
+ }
Vanilla-qwen4b-20b/checkpoint-17188/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67d469d450411055d454f462e379f69c74b39a771ef14ffc569589a61978117d
3
+ size 8411554440
Vanilla-qwen4b-20b/checkpoint-17188/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-17188/tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "extra_special_tokens": [
12
+ "<|im_start|>",
13
+ "<|im_end|>",
14
+ "<|object_ref_start|>",
15
+ "<|object_ref_end|>",
16
+ "<|box_start|>",
17
+ "<|box_end|>",
18
+ "<|quad_start|>",
19
+ "<|quad_end|>",
20
+ "<|vision_start|>",
21
+ "<|vision_end|>",
22
+ "<|vision_pad|>",
23
+ "<|image_pad|>",
24
+ "<|video_pad|>",
25
+ "<fact>",
26
+ "</fact>"
27
+ ],
28
+ "image_token": "<|image_pad|>",
29
+ "model_max_length": 32768,
30
+ "model_specific_special_tokens": {
31
+ "audio_bos_token": "<|audio_start|>",
32
+ "audio_eos_token": "<|audio_end|>",
33
+ "audio_token": "<|audio_pad|>",
34
+ "image_token": "<|image_pad|>",
35
+ "video_token": "<|video_pad|>",
36
+ "vision_bos_token": "<|vision_start|>",
37
+ "vision_eos_token": "<|vision_end|>"
38
+ },
39
+ "pad_token": "<|endoftext|>",
40
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null,
44
+ "video_token": "<|video_pad|>",
45
+ "vision_bos_token": "<|vision_start|>",
46
+ "vision_eos_token": "<|vision_end|>"
47
+ }
Vanilla-qwen4b-20b/checkpoint-1910/config.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3_5ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "attn_output_gate": true,
8
+ "bos_token_id": null,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 248044,
11
+ "full_attention_interval": 4,
12
+ "head_dim": 256,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 9216,
17
+ "layer_types": [
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "linear_attention",
21
+ "full_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "linear_attention",
25
+ "full_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "linear_attention",
29
+ "full_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "linear_attention",
33
+ "full_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "linear_attention",
37
+ "full_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "linear_attention",
41
+ "full_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "linear_attention",
45
+ "full_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "linear_attention",
49
+ "full_attention"
50
+ ],
51
+ "linear_conv_kernel_dim": 4,
52
+ "linear_key_head_dim": 128,
53
+ "linear_num_key_heads": 16,
54
+ "linear_num_value_heads": 32,
55
+ "linear_value_head_dim": 128,
56
+ "mamba_ssm_dtype": "float32",
57
+ "max_position_embeddings": 262144,
58
+ "mlp_only_layers": [],
59
+ "model_type": "qwen3_5_text",
60
+ "mtp_num_hidden_layers": 1,
61
+ "mtp_use_dedicated_embeddings": false,
62
+ "num_attention_heads": 16,
63
+ "num_hidden_layers": 32,
64
+ "num_key_value_heads": 4,
65
+ "pad_token_id": 248044,
66
+ "partial_rotary_factor": 0.25,
67
+ "rms_norm_eps": 1e-06,
68
+ "rope_parameters": {
69
+ "mrope_interleaved": true,
70
+ "mrope_section": [
71
+ 11,
72
+ 11,
73
+ 10
74
+ ],
75
+ "partial_rotary_factor": 0.25,
76
+ "rope_theta": 10000000,
77
+ "rope_type": "default"
78
+ },
79
+ "tie_word_embeddings": true,
80
+ "transformers_version": "5.6.2",
81
+ "use_cache": true,
82
+ "vocab_size": 248320
83
+ }
Vanilla-qwen4b-20b/checkpoint-1910/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 248044,
4
+ "output_attentions": false,
5
+ "output_hidden_states": false,
6
+ "pad_token_id": 248044,
7
+ "transformers_version": "5.6.2",
8
+ "use_cache": true
9
+ }
Vanilla-qwen4b-20b/checkpoint-1910/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebbecc5de3ed774dbfce3387c20cba60244884b3690bf9aa59c26a11769e5557
3
+ size 8411554440
Vanilla-qwen4b-20b/checkpoint-1910/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f8a9ce416e1039464b3e8c00e1e284f037b63350c952ce2d5f641690e3a9a1f
3
+ size 19989692
Vanilla-qwen4b-20b/checkpoint-1910/tokenizer_config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "audio_bos_token": "<|audio_start|>",
4
+ "audio_eos_token": "<|audio_end|>",
5
+ "audio_token": "<|audio_pad|>",
6
+ "backend": "tokenizers",
7
+ "bos_token": null,
8
+ "clean_up_tokenization_spaces": false,
9
+ "eos_token": "<|endoftext|>",
10
+ "errors": "replace",
11
+ "extra_special_tokens": [
12
+ "<|im_start|>",
13
+ "<|im_end|>",
14
+ "<|object_ref_start|>",
15
+ "<|object_ref_end|>",
16
+ "<|box_start|>",
17
+ "<|box_end|>",
18
+ "<|quad_start|>",
19
+ "<|quad_end|>",
20
+ "<|vision_start|>",
21
+ "<|vision_end|>",
22
+ "<|vision_pad|>",
23
+ "<|image_pad|>",
24
+ "<|video_pad|>",
25
+ "<fact>",
26
+ "</fact>"
27
+ ],
28
+ "image_token": "<|image_pad|>",
29
+ "model_max_length": 32768,
30
+ "model_specific_special_tokens": {
31
+ "audio_bos_token": "<|audio_start|>",
32
+ "audio_eos_token": "<|audio_end|>",
33
+ "audio_token": "<|audio_pad|>",
34
+ "image_token": "<|image_pad|>",
35
+ "video_token": "<|video_pad|>",
36
+ "vision_bos_token": "<|vision_start|>",
37
+ "vision_eos_token": "<|vision_end|>"
38
+ },
39
+ "pad_token": "<|endoftext|>",
40
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null,
44
+ "video_token": "<|video_pad|>",
45
+ "vision_bos_token": "<|vision_start|>",
46
+ "vision_eos_token": "<|vision_end|>"
47
+ }