Johnblick187 commited on
Commit
75b5510
·
verified ·
1 Parent(s): 8c5a8e2

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +26 -0
  2. blobs/03dd32f96f668553586b8b6b2fa821ba08672ae1 +115 -0
  3. blobs/0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687 +3 -0
  4. blobs/17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae +3 -0
  5. blobs/1ac848e36c5e24a7f31f64b5631eb8890ffe916f +117 -0
  6. blobs/20ef74aaea93d005bad8ce4e7aecd870b4c85c60 +115 -0
  7. blobs/211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71 +3 -0
  8. blobs/2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421 +3 -0
  9. blobs/39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8 +3 -0
  10. blobs/395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba +3 -0
  11. blobs/52373fe24473b1aa44333d318f578ae6bf04b49b +36 -0
  12. blobs/556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829 +3 -0
  13. blobs/56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f +3 -0
  14. blobs/5d5e4a4c418e93217947ca434c046d36ff634400 +12 -0
  15. blobs/60e155923df59cf410facf6dce3ddb14fac6b37d +31 -0
  16. blobs/69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d +3 -0
  17. blobs/6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057 +3 -0
  18. blobs/71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa +3 -0
  19. blobs/83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c +3 -0
  20. blobs/90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411 +3 -0
  21. blobs/977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd +3 -0
  22. blobs/9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0 +3 -0
  23. blobs/a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74 +3 -0
  24. blobs/adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e +3 -0
  25. blobs/af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56 +3 -0
  26. blobs/b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620 +3 -0
  27. blobs/b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee +3 -0
  28. blobs/bbfbbce7c8eaa0684336e3b64ce4a02bacdd6685 +110 -0
  29. blobs/bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685 +3 -0
  30. blobs/be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +3 -0
  31. blobs/c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8 +3 -0
  32. blobs/c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639 +3 -0
  33. blobs/dc13e5abfbc7bb1f8ca8e8c564cd2d1b10d1a273 +0 -0
  34. blobs/e6f3a82c69f4735fd65b222bc6af006d4bd23dcd +30 -0
  35. blobs/f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237 +3 -0
  36. blobs/fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce +3 -0
  37. refs/main +1 -0
  38. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/.gitattributes +36 -0
  39. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/README.md +110 -0
  40. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/chat_template.jinja +117 -0
  41. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/config.json +115 -0
  42. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/generation_config.json +12 -0
  43. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00002-of-00025.safetensors +3 -0
  44. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00003-of-00025.safetensors +3 -0
  45. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00004-of-00025.safetensors +3 -0
  46. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00008-of-00025.safetensors +3 -0
  47. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00016-of-00025.safetensors +3 -0
  48. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00023-of-00025.safetensors +3 -0
  49. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00025-of-00025.safetensors +3 -0
  50. snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model.safetensors.index.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,29 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ blobs/39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8 filter=lfs diff=lfs merge=lfs -text
37
+ blobs/a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74 filter=lfs diff=lfs merge=lfs -text
38
+ blobs/83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c filter=lfs diff=lfs merge=lfs -text
39
+ blobs/69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d filter=lfs diff=lfs merge=lfs -text
40
+ blobs/c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639 filter=lfs diff=lfs merge=lfs -text
41
+ blobs/2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421 filter=lfs diff=lfs merge=lfs -text
42
+ blobs/adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e filter=lfs diff=lfs merge=lfs -text
43
+ blobs/977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd filter=lfs diff=lfs merge=lfs -text
44
+ blobs/9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0 filter=lfs diff=lfs merge=lfs -text
45
+ blobs/56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f filter=lfs diff=lfs merge=lfs -text
46
+ blobs/211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71 filter=lfs diff=lfs merge=lfs -text
47
+ blobs/17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae filter=lfs diff=lfs merge=lfs -text
48
+ blobs/556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829 filter=lfs diff=lfs merge=lfs -text
49
+ blobs/c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8 filter=lfs diff=lfs merge=lfs -text
50
+ blobs/be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 filter=lfs diff=lfs merge=lfs -text
51
+ blobs/bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685 filter=lfs diff=lfs merge=lfs -text
52
+ blobs/395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba filter=lfs diff=lfs merge=lfs -text
53
+ blobs/b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620 filter=lfs diff=lfs merge=lfs -text
54
+ blobs/f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237 filter=lfs diff=lfs merge=lfs -text
55
+ blobs/71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa filter=lfs diff=lfs merge=lfs -text
56
+ blobs/90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411 filter=lfs diff=lfs merge=lfs -text
57
+ blobs/fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce filter=lfs diff=lfs merge=lfs -text
58
+ blobs/0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687 filter=lfs diff=lfs merge=lfs -text
59
+ blobs/af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56 filter=lfs diff=lfs merge=lfs -text
60
+ blobs/b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee filter=lfs diff=lfs merge=lfs -text
61
+ blobs/6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057 filter=lfs diff=lfs merge=lfs -text
blobs/03dd32f96f668553586b8b6b2fa821ba08672ae1 ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3NextForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0,
7
+ "bos_token_id": null,
8
+ "decoder_sparse_step": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 151645,
11
+ "head_dim": 256,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 2048,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 5120,
16
+ "layer_types": [
17
+ "linear_attention",
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "full_attention",
21
+ "linear_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "full_attention",
25
+ "linear_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "full_attention",
29
+ "linear_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "full_attention",
33
+ "linear_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "full_attention",
37
+ "linear_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "full_attention",
41
+ "linear_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "full_attention",
45
+ "linear_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "full_attention",
49
+ "linear_attention",
50
+ "linear_attention",
51
+ "linear_attention",
52
+ "full_attention",
53
+ "linear_attention",
54
+ "linear_attention",
55
+ "linear_attention",
56
+ "full_attention",
57
+ "linear_attention",
58
+ "linear_attention",
59
+ "linear_attention",
60
+ "full_attention",
61
+ "linear_attention",
62
+ "linear_attention",
63
+ "linear_attention",
64
+ "full_attention"
65
+ ],
66
+ "linear_conv_kernel_dim": 4,
67
+ "linear_key_head_dim": 128,
68
+ "linear_num_key_heads": 16,
69
+ "linear_num_value_heads": 32,
70
+ "linear_value_head_dim": 128,
71
+ "max_position_embeddings": 262144,
72
+ "merge_args": {
73
+ "batch_size": 2,
74
+ "cache_dir": null,
75
+ "dataset": "c4+math+code",
76
+ "download": false,
77
+ "group_size": 16,
78
+ "grouping": "ream",
79
+ "merge_size": 256,
80
+ "merging": "logits+weights",
81
+ "mix_ratio": "0.0,0.3,0.7",
82
+ "model": "/root/.cache/huggingface/hub/models--samuelcardillo--Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled/snapshots/1a298f7f385aa744169db630a16d28e56b5566ff",
83
+ "mtp_safe_tensors": null,
84
+ "no_gate_output": false,
85
+ "no_gated_sim": false,
86
+ "no_sequential": false,
87
+ "saliency": "reap",
88
+ "save_path": "./samuel-ream-256"
89
+ },
90
+ "mlp_only_layers": [],
91
+ "model_type": "qwen3_next",
92
+ "moe_intermediate_size": 512,
93
+ "norm_topk_prob": true,
94
+ "num_attention_heads": 16,
95
+ "num_experts": 256,
96
+ "num_experts_per_tok": 10,
97
+ "num_hidden_layers": 48,
98
+ "num_key_value_heads": 2,
99
+ "output_router_logits": false,
100
+ "pad_token_id": 151643,
101
+ "partial_rotary_factor": 0.25,
102
+ "rms_norm_eps": 1e-06,
103
+ "rope_parameters": {
104
+ "partial_rotary_factor": 0.25,
105
+ "rope_theta": 5000000,
106
+ "rope_type": "default"
107
+ },
108
+ "router_aux_loss_coef": 0.001,
109
+ "shared_expert_intermediate_size": 512,
110
+ "tie_word_embeddings": false,
111
+ "transformers_version": "5.6.0",
112
+ "use_cache": false,
113
+ "use_sliding_window": false,
114
+ "vocab_size": 151936
115
+ }
blobs/0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687
3
+ size 3425524216
blobs/17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae
3
+ size 3303560376
blobs/1ac848e36c5e24a7f31f64b5631eb8890ffe916f ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% macro render_extra_keys(json_dict, handled_keys) %}
2
+ {%- if json_dict is mapping %}
3
+ {%- for json_key in json_dict if json_key not in handled_keys %}
4
+ {%- if json_dict[json_key] is string %}
5
+ {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
6
+ {%- else %}
7
+ {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
8
+ {%- endif %}
9
+ {%- endfor %}
10
+ {%- endif %}
11
+ {%- endmacro %}
12
+
13
+ {%- if messages[0]["role"] == "system" %}
14
+ {%- set system_message = messages[0]["content"] %}
15
+ {%- set loop_messages = messages[1:] %}
16
+ {%- else %}
17
+ {%- set loop_messages = messages %}
18
+ {%- endif %}
19
+
20
+ {%- if not tools is defined %}
21
+ {%- set tools = [] %}
22
+ {%- endif %}
23
+
24
+ {%- if system_message is defined %}
25
+ {{- "<|im_start|>system\n" + system_message }}
26
+ {%- else %}
27
+ {%- if tools is iterable and tools | length > 0 %}
28
+ {{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
29
+ {%- endif %}
30
+ {%- endif %}
31
+ {%- if tools is iterable and tools | length > 0 %}
32
+ {{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
33
+ {{- "<tools>" }}
34
+ {%- for tool in tools %}
35
+ {%- if tool.function is defined %}
36
+ {%- set tool = tool.function %}
37
+ {%- endif %}
38
+ {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
39
+ {%- if tool.description is defined %}
40
+ {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
41
+ {%- endif %}
42
+ {{- '\n<parameters>' }}
43
+ {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
44
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
45
+ {{- '\n<parameter>' }}
46
+ {{- '\n<name>' ~ param_name ~ '</name>' }}
47
+ {%- if param_fields.type is defined %}
48
+ {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
49
+ {%- endif %}
50
+ {%- if param_fields.description is defined %}
51
+ {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
52
+ {%- endif %}
53
+ {%- set handled_keys = ['name', 'type', 'description'] %}
54
+ {{- render_extra_keys(param_fields, handled_keys) }}
55
+ {{- '\n</parameter>' }}
56
+ {%- endfor %}
57
+ {%- endif %}
58
+ {%- set handled_keys = ['type', 'properties'] %}
59
+ {{- render_extra_keys(tool.parameters, handled_keys) }}
60
+ {{- '\n</parameters>' }}
61
+ {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
62
+ {{- render_extra_keys(tool, handled_keys) }}
63
+ {{- '\n</function>' }}
64
+ {%- endfor %}
65
+ {{- "\n</tools>" }}
66
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
67
+ {%- endif %}
68
+ {%- if system_message is defined %}
69
+ {{- '<|im_end|>\n' }}
70
+ {%- else %}
71
+ {%- if tools is iterable and tools | length > 0 %}
72
+ {{- '<|im_end|>\n' }}
73
+ {%- endif %}
74
+ {%- endif %}
75
+ {%- for message in loop_messages %}
76
+ {%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
77
+ {{- '<|im_start|>' + message.role }}
78
+ {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
79
+ {{- '\n' + message.content | trim + '\n' }}
80
+ {%- endif %}
81
+ {%- for tool_call in message.tool_calls %}
82
+ {%- if tool_call.function is defined %}
83
+ {%- set tool_call = tool_call.function %}
84
+ {%- endif %}
85
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
86
+ {%- if tool_call.arguments is defined %}
87
+ {%- for args_name, args_value in tool_call.arguments|items %}
88
+ {{- '<parameter=' + args_name + '>\n' }}
89
+ {%- set args_value = args_value if args_value is string else args_value | tojson | safe %}
90
+ {{- args_value }}
91
+ {{- '\n</parameter>\n' }}
92
+ {%- endfor %}
93
+ {%- endif %}
94
+ {{- '</function>\n</tool_call>' }}
95
+ {%- endfor %}
96
+ {{- '<|im_end|>\n' }}
97
+ {%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
98
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
99
+ {%- elif message.role == "tool" %}
100
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
101
+ {{- '<|im_start|>user' }}
102
+ {%- endif %}
103
+ {{- '\n<tool_response>\n' }}
104
+ {{- message.content }}
105
+ {{- '\n</tool_response>' }}
106
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
107
+ {{- '<|im_end|>\n' }}
108
+ {%- elif loop.last %}
109
+ {{- '<|im_end|>\n' }}
110
+ {%- endif %}
111
+ {%- else %}
112
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
113
+ {%- endif %}
114
+ {%- endfor %}
115
+ {%- if add_generation_prompt %}
116
+ {{- '<|im_start|>assistant\n' }}
117
+ {%- endif %}
blobs/20ef74aaea93d005bad8ce4e7aecd870b4c85c60 ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3NextForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0,
7
+ "bos_token_id": null,
8
+ "decoder_sparse_step": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 151645,
11
+ "head_dim": 256,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 2048,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 5120,
16
+ "layer_types": [
17
+ "linear_attention",
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "full_attention",
21
+ "linear_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "full_attention",
25
+ "linear_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "full_attention",
29
+ "linear_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "full_attention",
33
+ "linear_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "full_attention",
37
+ "linear_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "full_attention",
41
+ "linear_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "full_attention",
45
+ "linear_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "full_attention",
49
+ "linear_attention",
50
+ "linear_attention",
51
+ "linear_attention",
52
+ "full_attention",
53
+ "linear_attention",
54
+ "linear_attention",
55
+ "linear_attention",
56
+ "full_attention",
57
+ "linear_attention",
58
+ "linear_attention",
59
+ "linear_attention",
60
+ "full_attention",
61
+ "linear_attention",
62
+ "linear_attention",
63
+ "linear_attention",
64
+ "full_attention"
65
+ ],
66
+ "linear_conv_kernel_dim": 4,
67
+ "linear_key_head_dim": 128,
68
+ "linear_num_key_heads": 16,
69
+ "linear_num_value_heads": 32,
70
+ "linear_value_head_dim": 128,
71
+ "max_position_embeddings": 32768,
72
+ "merge_args": {
73
+ "batch_size": 2,
74
+ "cache_dir": null,
75
+ "dataset": "c4+math+code",
76
+ "download": false,
77
+ "group_size": 16,
78
+ "grouping": "ream",
79
+ "merge_size": 256,
80
+ "merging": "logits+weights",
81
+ "mix_ratio": "0.0,0.3,0.7",
82
+ "model": "/root/.cache/huggingface/hub/models--samuelcardillo--Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled/snapshots/1a298f7f385aa744169db630a16d28e56b5566ff",
83
+ "mtp_safe_tensors": null,
84
+ "no_gate_output": false,
85
+ "no_gated_sim": false,
86
+ "no_sequential": false,
87
+ "saliency": "reap",
88
+ "save_path": "./samuel-ream-256"
89
+ },
90
+ "mlp_only_layers": [],
91
+ "model_type": "qwen3_next",
92
+ "moe_intermediate_size": 512,
93
+ "norm_topk_prob": true,
94
+ "num_attention_heads": 16,
95
+ "num_experts": 256,
96
+ "num_experts_per_tok": 10,
97
+ "num_hidden_layers": 48,
98
+ "num_key_value_heads": 2,
99
+ "output_router_logits": false,
100
+ "pad_token_id": 151643,
101
+ "partial_rotary_factor": 0.25,
102
+ "rms_norm_eps": 1e-06,
103
+ "rope_parameters": {
104
+ "partial_rotary_factor": 0.25,
105
+ "rope_theta": 5000000,
106
+ "rope_type": "default"
107
+ },
108
+ "router_aux_loss_coef": 0.001,
109
+ "shared_expert_intermediate_size": 512,
110
+ "tie_word_embeddings": false,
111
+ "transformers_version": "5.6.0",
112
+ "use_cache": false,
113
+ "use_sliding_window": false,
114
+ "vocab_size": 151936
115
+ }
blobs/211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71
3
+ size 3425525792
blobs/2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421
3
+ size 3534503480
blobs/39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8
3
+ size 3303559928
blobs/395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba
3
+ size 3425524216
blobs/52373fe24473b1aa44333d318f578ae6bf04b49b ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
blobs/556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829
3
+ size 3425525792
blobs/56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f
3
+ size 3425525792
blobs/5d5e4a4c418e93217947ca434c046d36ff634400 ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "temperature": 1.0,
9
+ "top_k": 40,
10
+ "top_p": 0.95,
11
+ "transformers_version": "5.6.0"
12
+ }
blobs/60e155923df59cf410facf6dce3ddb14fac6b37d ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": true,
24
+ "local_files_only": false,
25
+ "model_max_length": 32768,
26
+ "pad_token": "<|im_end|>",
27
+ "split_special_tokens": false,
28
+ "tokenizer_class": "Qwen2Tokenizer",
29
+ "unk_token": null,
30
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
31
+ }
blobs/69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d
3
+ size 3303560376
blobs/6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057
3
+ size 3425525792
blobs/71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa
3
+ size 3303558816
blobs/83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c
3
+ size 3303560376
blobs/90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411
3
+ size 3303560376
blobs/977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd
3
+ size 3303560376
blobs/9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0
3
+ size 3425525792
blobs/a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74
3
+ size 3303560376
blobs/adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e
3
+ size 1135685400
blobs/af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56
3
+ size 3303560376
blobs/b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620
3
+ size 3425525792
blobs/b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee
3
+ size 3303558816
blobs/bbfbbce7c8eaa0684336e3b64ce4a02bacdd6685 ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ language:
4
+ - en
5
+ base_model:
6
+ - samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled
7
+ pipeline_tag: text-generation
8
+ library_name: transformers
9
+ tags:
10
+ - qwen3next
11
+ - Code
12
+ - REAM
13
+ - custom-merge
14
+ ---
15
+
16
+ # Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B
17
+
18
+ A REAM-merged version of [samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled](https://huggingface.co/samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled), reducing 512 experts to 256 (50% reduction) using [REAM (Router-weighted Expert Activation Merging)](https://github.com/SamsungSAILMontreal/ream).
19
+
20
+ ## Status: Experimental
21
+
22
+ To my knowledge this is the **first publicly released REAM model at 50% expert reduction on the `qwen3_next` architecture**. Prior published REAM Coder-Next models (e.g. [bknyaz/Qwen3-Coder-Next-REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM)) target 25% reduction, which the REAM paper authors recommend for production use.
23
+
24
+ **The REAM authors explicitly note that merging quality degrades at 50% compared to 25%, and the REAP authors empirically demonstrate that pruning (REAP) outperforms merging (REAM) at 50% reduction on generative tasks.** This release is an experimental artifact rather than a recommended production model. Comprehensive evaluation has not been run.
25
+
26
+ ## Model Details
27
+
28
+ - **Architecture:** `qwen3_next` (hybrid Gated DeltaNet + Attention + MoE)
29
+ - **Parameters:** ~40B total, 3B active per forward pass
30
+ - **Layers:** 48
31
+ - **Experts:** 256 (merged from 512)
32
+ - **Hidden dim:** 2048
33
+ - **moe_intermediate_size:** 512
34
+ - **Precision:** bf16
35
+ - **Source:** Samuel Cardillo’s full fine-tune of Qwen3-Coder-Next on Claude Opus 4.6 reasoning distillation data
36
+
37
+ ## REAM Configuration Used
38
+
39
+ ```
40
+ saliency: reap
41
+ merging: logits+weights
42
+ grouping: ream
43
+ merge_size: 256 (50% of original 512)
44
+ group_size: 16
45
+ calibration: c4+math+code (default mix 0.0:0.3:0.7)
46
+ batch_size: 2
47
+ sequential: True
48
+ ```
49
+
50
+ Per-layer reduction confirmed cleanly during the run: each layer’s 512 experts merged to 256 with the expected 2:1 parameter reduction (e.g. 1.61B → 808M expert params per layer).
51
+
52
+ ## Intended Use
53
+
54
+ - Research into expert merging methods on hybrid MoE architectures
55
+ - Comparison baseline for staged-REAM or other quality-preservation techniques at high compression ratios
56
+ - Base for further fine-tuning experiments where 40B-class size is needed
57
+
58
+ ## Not Recommended For
59
+
60
+ - Production deployment without further evaluation. Use [bknyaz’s 25% REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM) or [lovedheart’s REAP-40B](https://huggingface.co/lovedheart/Qwen3-Coder-Next-REAP-40B-A3B) instead.
61
+ - Code generation at production quality without verification on your specific task.
62
+ - Comparison against the original Samuel 80B without acknowledging the 50% capacity reduction.
63
+
64
+ ## Limitations
65
+
66
+ - 50% expert reduction necessarily loses capability compared to the 512-expert source.
67
+ - REAM at 50% is documented as suboptimal compared to REAP at 50% for generative tasks (Cerebras 2025).
68
+ - No benchmarks have been run on this specific checkpoint.
69
+ - Inherits all biases and limitations from Samuel’s source model and the original Qwen3-Coder-Next pretraining.
70
+
71
+ ## Loading
72
+
73
+ ```python
74
+ from transformers import AutoModelForCausalLM, AutoTokenizer
75
+
76
+ model = AutoModelForCausalLM.from_pretrained(
77
+ "Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
78
+ torch_dtype="auto",
79
+ device_map="auto",
80
+ trust_remote_code=True,
81
+ )
82
+ tokenizer = AutoTokenizer.from_pretrained(
83
+ "Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
84
+ trust_remote_code=True,
85
+ )
86
+ ```
87
+
88
+ ~80GB VRAM needed in bf16, ~20GB at 4-bit.
89
+
90
+ ## Acknowledgements
91
+
92
+ - [Samuel Cardillo](https://huggingface.co/samuelcardillo) for the source fine-tune
93
+ - [Samsung SAIL Montreal](https://github.com/SamsungSAILMontreal/ream) and [Boris Knyazev](https://bknyaz.github.io/blog/2026/moe/) for REAM
94
+ - [Cerebras Research](https://github.com/CerebrasResearch/reap) for REAP and the comparative analysis
95
+ - [Qwen team](https://huggingface.co/Qwen) for Qwen3-Coder-Next
96
+ - [lovedheart](https://huggingface.co/lovedheart) for prior REAP work on this architecture
97
+
98
+ ## Training Compute
99
+
100
+ Single H200 NVL on RunPod. ~250GB system RAM used during merge (model loaded `device_map='cpu'` per REAM script default). ~3.5 hours runtime. Sequential layer-by-layer merge mode.
101
+
102
+ ## References
103
+
104
+ - REAM paper: https://arxiv.org/abs/2604.04356
105
+ - REAP paper: https://arxiv.org/abs/2510.13999
106
+ - bknyaz blog: https://bknyaz.github.io/blog/2026/moe/
107
+
108
+ ## Disclaimer
109
+
110
+ The “Claude Opus 4.6” attribution in the name is propagated from Samuel’s source model name. This model and its source were not produced by, endorsed by, or affiliated with Anthropic. Use of this model should comply with Anthropic’s terms regarding training on Claude outputs if applicable to your use case.
blobs/bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685
3
+ size 3425525792
blobs/be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
3
+ size 11422650
blobs/c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8
3
+ size 3303560376
blobs/c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639
3
+ size 3425525792
blobs/dc13e5abfbc7bb1f8ca8e8c564cd2d1b10d1a273 ADDED
The diff for this file is too large to render. See raw diff
 
blobs/e6f3a82c69f4735fd65b222bc6af006d4bd23dcd ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": true,
24
+ "local_files_only": false,
25
+ "model_max_length": 1048576,
26
+ "pad_token": "<|endoftext|>",
27
+ "split_special_tokens": false,
28
+ "tokenizer_class": "Qwen2Tokenizer",
29
+ "unk_token": null
30
+ }
blobs/f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237
3
+ size 3425525792
blobs/fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce
3
+ size 3303560376
refs/main ADDED
@@ -0,0 +1 @@
 
 
1
+ 060d033c323b7bbc8cc0e7c88621678ca1c648ca
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/.gitattributes ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/README.md ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ language:
4
+ - en
5
+ base_model:
6
+ - samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled
7
+ pipeline_tag: text-generation
8
+ library_name: transformers
9
+ tags:
10
+ - qwen3next
11
+ - Code
12
+ - REAM
13
+ - custom-merge
14
+ ---
15
+
16
+ # Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B
17
+
18
+ A REAM-merged version of [samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled](https://huggingface.co/samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled), reducing 512 experts to 256 (50% reduction) using [REAM (Router-weighted Expert Activation Merging)](https://github.com/SamsungSAILMontreal/ream).
19
+
20
+ ## Status: Experimental
21
+
22
+ To my knowledge this is the **first publicly released REAM model at 50% expert reduction on the `qwen3_next` architecture**. Prior published REAM Coder-Next models (e.g. [bknyaz/Qwen3-Coder-Next-REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM)) target 25% reduction, which the REAM paper authors recommend for production use.
23
+
24
+ **The REAM authors explicitly note that merging quality degrades at 50% compared to 25%, and the REAP authors empirically demonstrate that pruning (REAP) outperforms merging (REAM) at 50% reduction on generative tasks.** This release is an experimental artifact rather than a recommended production model. Comprehensive evaluation has not been run.
25
+
26
+ ## Model Details
27
+
28
+ - **Architecture:** `qwen3_next` (hybrid Gated DeltaNet + Attention + MoE)
29
+ - **Parameters:** ~40B total, 3B active per forward pass
30
+ - **Layers:** 48
31
+ - **Experts:** 256 (merged from 512)
32
+ - **Hidden dim:** 2048
33
+ - **moe_intermediate_size:** 512
34
+ - **Precision:** bf16
35
+ - **Source:** Samuel Cardillo’s full fine-tune of Qwen3-Coder-Next on Claude Opus 4.6 reasoning distillation data
36
+
37
+ ## REAM Configuration Used
38
+
39
+ ```
40
+ saliency: reap
41
+ merging: logits+weights
42
+ grouping: ream
43
+ merge_size: 256 (50% of original 512)
44
+ group_size: 16
45
+ calibration: c4+math+code (default mix 0.0:0.3:0.7)
46
+ batch_size: 2
47
+ sequential: True
48
+ ```
49
+
50
+ Per-layer reduction confirmed cleanly during the run: each layer’s 512 experts merged to 256 with the expected 2:1 parameter reduction (e.g. 1.61B → 808M expert params per layer).
51
+
52
+ ## Intended Use
53
+
54
+ - Research into expert merging methods on hybrid MoE architectures
55
+ - Comparison baseline for staged-REAM or other quality-preservation techniques at high compression ratios
56
+ - Base for further fine-tuning experiments where 40B-class size is needed
57
+
58
+ ## Not Recommended For
59
+
60
+ - Production deployment without further evaluation. Use [bknyaz’s 25% REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM) or [lovedheart’s REAP-40B](https://huggingface.co/lovedheart/Qwen3-Coder-Next-REAP-40B-A3B) instead.
61
+ - Code generation at production quality without verification on your specific task.
62
+ - Comparison against the original Samuel 80B without acknowledging the 50% capacity reduction.
63
+
64
+ ## Limitations
65
+
66
+ - 50% expert reduction necessarily loses capability compared to the 512-expert source.
67
+ - REAM at 50% is documented as suboptimal compared to REAP at 50% for generative tasks (Cerebras 2025).
68
+ - No benchmarks have been run on this specific checkpoint.
69
+ - Inherits all biases and limitations from Samuel’s source model and the original Qwen3-Coder-Next pretraining.
70
+
71
+ ## Loading
72
+
73
+ ```python
74
+ from transformers import AutoModelForCausalLM, AutoTokenizer
75
+
76
+ model = AutoModelForCausalLM.from_pretrained(
77
+ "Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
78
+ torch_dtype="auto",
79
+ device_map="auto",
80
+ trust_remote_code=True,
81
+ )
82
+ tokenizer = AutoTokenizer.from_pretrained(
83
+ "Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
84
+ trust_remote_code=True,
85
+ )
86
+ ```
87
+
88
+ ~80GB VRAM needed in bf16, ~20GB at 4-bit.
89
+
90
+ ## Acknowledgements
91
+
92
+ - [Samuel Cardillo](https://huggingface.co/samuelcardillo) for the source fine-tune
93
+ - [Samsung SAIL Montreal](https://github.com/SamsungSAILMontreal/ream) and [Boris Knyazev](https://bknyaz.github.io/blog/2026/moe/) for REAM
94
+ - [Cerebras Research](https://github.com/CerebrasResearch/reap) for REAP and the comparative analysis
95
+ - [Qwen team](https://huggingface.co/Qwen) for Qwen3-Coder-Next
96
+ - [lovedheart](https://huggingface.co/lovedheart) for prior REAP work on this architecture
97
+
98
+ ## Training Compute
99
+
100
+ Single H200 NVL on RunPod. ~250GB system RAM used during merge (model loaded `device_map='cpu'` per REAM script default). ~3.5 hours runtime. Sequential layer-by-layer merge mode.
101
+
102
+ ## References
103
+
104
+ - REAM paper: https://arxiv.org/abs/2604.04356
105
+ - REAP paper: https://arxiv.org/abs/2510.13999
106
+ - bknyaz blog: https://bknyaz.github.io/blog/2026/moe/
107
+
108
+ ## Disclaimer
109
+
110
+ The “Claude Opus 4.6” attribution in the name is propagated from Samuel’s source model name. This model and its source were not produced by, endorsed by, or affiliated with Anthropic. Use of this model should comply with Anthropic’s terms regarding training on Claude outputs if applicable to your use case.
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/chat_template.jinja ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% macro render_extra_keys(json_dict, handled_keys) %}
2
+ {%- if json_dict is mapping %}
3
+ {%- for json_key in json_dict if json_key not in handled_keys %}
4
+ {%- if json_dict[json_key] is string %}
5
+ {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
6
+ {%- else %}
7
+ {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
8
+ {%- endif %}
9
+ {%- endfor %}
10
+ {%- endif %}
11
+ {%- endmacro %}
12
+
13
+ {%- if messages[0]["role"] == "system" %}
14
+ {%- set system_message = messages[0]["content"] %}
15
+ {%- set loop_messages = messages[1:] %}
16
+ {%- else %}
17
+ {%- set loop_messages = messages %}
18
+ {%- endif %}
19
+
20
+ {%- if not tools is defined %}
21
+ {%- set tools = [] %}
22
+ {%- endif %}
23
+
24
+ {%- if system_message is defined %}
25
+ {{- "<|im_start|>system\n" + system_message }}
26
+ {%- else %}
27
+ {%- if tools is iterable and tools | length > 0 %}
28
+ {{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
29
+ {%- endif %}
30
+ {%- endif %}
31
+ {%- if tools is iterable and tools | length > 0 %}
32
+ {{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
33
+ {{- "<tools>" }}
34
+ {%- for tool in tools %}
35
+ {%- if tool.function is defined %}
36
+ {%- set tool = tool.function %}
37
+ {%- endif %}
38
+ {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
39
+ {%- if tool.description is defined %}
40
+ {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
41
+ {%- endif %}
42
+ {{- '\n<parameters>' }}
43
+ {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
44
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
45
+ {{- '\n<parameter>' }}
46
+ {{- '\n<name>' ~ param_name ~ '</name>' }}
47
+ {%- if param_fields.type is defined %}
48
+ {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
49
+ {%- endif %}
50
+ {%- if param_fields.description is defined %}
51
+ {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
52
+ {%- endif %}
53
+ {%- set handled_keys = ['name', 'type', 'description'] %}
54
+ {{- render_extra_keys(param_fields, handled_keys) }}
55
+ {{- '\n</parameter>' }}
56
+ {%- endfor %}
57
+ {%- endif %}
58
+ {%- set handled_keys = ['type', 'properties'] %}
59
+ {{- render_extra_keys(tool.parameters, handled_keys) }}
60
+ {{- '\n</parameters>' }}
61
+ {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
62
+ {{- render_extra_keys(tool, handled_keys) }}
63
+ {{- '\n</function>' }}
64
+ {%- endfor %}
65
+ {{- "\n</tools>" }}
66
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
67
+ {%- endif %}
68
+ {%- if system_message is defined %}
69
+ {{- '<|im_end|>\n' }}
70
+ {%- else %}
71
+ {%- if tools is iterable and tools | length > 0 %}
72
+ {{- '<|im_end|>\n' }}
73
+ {%- endif %}
74
+ {%- endif %}
75
+ {%- for message in loop_messages %}
76
+ {%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
77
+ {{- '<|im_start|>' + message.role }}
78
+ {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
79
+ {{- '\n' + message.content | trim + '\n' }}
80
+ {%- endif %}
81
+ {%- for tool_call in message.tool_calls %}
82
+ {%- if tool_call.function is defined %}
83
+ {%- set tool_call = tool_call.function %}
84
+ {%- endif %}
85
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
86
+ {%- if tool_call.arguments is defined %}
87
+ {%- for args_name, args_value in tool_call.arguments|items %}
88
+ {{- '<parameter=' + args_name + '>\n' }}
89
+ {%- set args_value = args_value if args_value is string else args_value | tojson | safe %}
90
+ {{- args_value }}
91
+ {{- '\n</parameter>\n' }}
92
+ {%- endfor %}
93
+ {%- endif %}
94
+ {{- '</function>\n</tool_call>' }}
95
+ {%- endfor %}
96
+ {{- '<|im_end|>\n' }}
97
+ {%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
98
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
99
+ {%- elif message.role == "tool" %}
100
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
101
+ {{- '<|im_start|>user' }}
102
+ {%- endif %}
103
+ {{- '\n<tool_response>\n' }}
104
+ {{- message.content }}
105
+ {{- '\n</tool_response>' }}
106
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
107
+ {{- '<|im_end|>\n' }}
108
+ {%- elif loop.last %}
109
+ {{- '<|im_end|>\n' }}
110
+ {%- endif %}
111
+ {%- else %}
112
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
113
+ {%- endif %}
114
+ {%- endfor %}
115
+ {%- if add_generation_prompt %}
116
+ {{- '<|im_start|>assistant\n' }}
117
+ {%- endif %}
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/config.json ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3NextForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0,
7
+ "bos_token_id": null,
8
+ "decoder_sparse_step": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 151645,
11
+ "head_dim": 256,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 2048,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 5120,
16
+ "layer_types": [
17
+ "linear_attention",
18
+ "linear_attention",
19
+ "linear_attention",
20
+ "full_attention",
21
+ "linear_attention",
22
+ "linear_attention",
23
+ "linear_attention",
24
+ "full_attention",
25
+ "linear_attention",
26
+ "linear_attention",
27
+ "linear_attention",
28
+ "full_attention",
29
+ "linear_attention",
30
+ "linear_attention",
31
+ "linear_attention",
32
+ "full_attention",
33
+ "linear_attention",
34
+ "linear_attention",
35
+ "linear_attention",
36
+ "full_attention",
37
+ "linear_attention",
38
+ "linear_attention",
39
+ "linear_attention",
40
+ "full_attention",
41
+ "linear_attention",
42
+ "linear_attention",
43
+ "linear_attention",
44
+ "full_attention",
45
+ "linear_attention",
46
+ "linear_attention",
47
+ "linear_attention",
48
+ "full_attention",
49
+ "linear_attention",
50
+ "linear_attention",
51
+ "linear_attention",
52
+ "full_attention",
53
+ "linear_attention",
54
+ "linear_attention",
55
+ "linear_attention",
56
+ "full_attention",
57
+ "linear_attention",
58
+ "linear_attention",
59
+ "linear_attention",
60
+ "full_attention",
61
+ "linear_attention",
62
+ "linear_attention",
63
+ "linear_attention",
64
+ "full_attention"
65
+ ],
66
+ "linear_conv_kernel_dim": 4,
67
+ "linear_key_head_dim": 128,
68
+ "linear_num_key_heads": 16,
69
+ "linear_num_value_heads": 32,
70
+ "linear_value_head_dim": 128,
71
+ "max_position_embeddings": 32768,
72
+ "merge_args": {
73
+ "batch_size": 2,
74
+ "cache_dir": null,
75
+ "dataset": "c4+math+code",
76
+ "download": false,
77
+ "group_size": 16,
78
+ "grouping": "ream",
79
+ "merge_size": 256,
80
+ "merging": "logits+weights",
81
+ "mix_ratio": "0.0,0.3,0.7",
82
+ "model": "/root/.cache/huggingface/hub/models--samuelcardillo--Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled/snapshots/1a298f7f385aa744169db630a16d28e56b5566ff",
83
+ "mtp_safe_tensors": null,
84
+ "no_gate_output": false,
85
+ "no_gated_sim": false,
86
+ "no_sequential": false,
87
+ "saliency": "reap",
88
+ "save_path": "./samuel-ream-256"
89
+ },
90
+ "mlp_only_layers": [],
91
+ "model_type": "qwen3_next",
92
+ "moe_intermediate_size": 512,
93
+ "norm_topk_prob": true,
94
+ "num_attention_heads": 16,
95
+ "num_experts": 256,
96
+ "num_experts_per_tok": 10,
97
+ "num_hidden_layers": 48,
98
+ "num_key_value_heads": 2,
99
+ "output_router_logits": false,
100
+ "pad_token_id": 151643,
101
+ "partial_rotary_factor": 0.25,
102
+ "rms_norm_eps": 1e-06,
103
+ "rope_parameters": {
104
+ "partial_rotary_factor": 0.25,
105
+ "rope_theta": 5000000,
106
+ "rope_type": "default"
107
+ },
108
+ "router_aux_loss_coef": 0.001,
109
+ "shared_expert_intermediate_size": 512,
110
+ "tie_word_embeddings": false,
111
+ "transformers_version": "5.6.0",
112
+ "use_cache": false,
113
+ "use_sliding_window": false,
114
+ "vocab_size": 151936
115
+ }
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "temperature": 1.0,
9
+ "top_k": 40,
10
+ "top_p": 0.95,
11
+ "transformers_version": "5.6.0"
12
+ }
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00002-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee
3
+ size 3303558816
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00003-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687
3
+ size 3425524216
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00004-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa
3
+ size 3303558816
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00008-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74
3
+ size 3303560376
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00016-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411
3
+ size 3303560376
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00023-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71
3
+ size 3425525792
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model-00025-of-00025.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e
3
+ size 1135685400
snapshots/060d033c323b7bbc8cc0e7c88621678ca1c648ca/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff