Delete blobs
Browse files- blobs/03dd32f96f668553586b8b6b2fa821ba08672ae1 +0 -115
- blobs/0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687 +0 -3
- blobs/17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae +0 -3
- blobs/1ac848e36c5e24a7f31f64b5631eb8890ffe916f +0 -117
- blobs/20ef74aaea93d005bad8ce4e7aecd870b4c85c60 +0 -115
- blobs/211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71 +0 -3
- blobs/2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421 +0 -3
- blobs/39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8 +0 -3
- blobs/395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba +0 -3
- blobs/52373fe24473b1aa44333d318f578ae6bf04b49b +0 -36
- blobs/556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829 +0 -3
- blobs/56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f +0 -3
- blobs/5d5e4a4c418e93217947ca434c046d36ff634400 +0 -12
- blobs/60e155923df59cf410facf6dce3ddb14fac6b37d +0 -31
- blobs/69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d +0 -3
- blobs/6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057 +0 -3
- blobs/71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa +0 -3
- blobs/83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c +0 -3
- blobs/90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411 +0 -3
- blobs/977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd +0 -3
- blobs/9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0 +0 -3
- blobs/a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74 +0 -3
- blobs/adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e +0 -3
- blobs/af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56 +0 -3
- blobs/b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620 +0 -3
- blobs/b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee +0 -3
- blobs/bbfbbce7c8eaa0684336e3b64ce4a02bacdd6685 +0 -110
- blobs/bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685 +0 -3
- blobs/be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +0 -3
- blobs/c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8 +0 -3
- blobs/c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639 +0 -3
- blobs/dc13e5abfbc7bb1f8ca8e8c564cd2d1b10d1a273 +0 -0
- blobs/e6f3a82c69f4735fd65b222bc6af006d4bd23dcd +0 -30
- blobs/f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237 +0 -3
- blobs/fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce +0 -3
blobs/03dd32f96f668553586b8b6b2fa821ba08672ae1
DELETED
|
@@ -1,115 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"architectures": [
|
| 3 |
-
"Qwen3NextForCausalLM"
|
| 4 |
-
],
|
| 5 |
-
"attention_bias": false,
|
| 6 |
-
"attention_dropout": 0,
|
| 7 |
-
"bos_token_id": null,
|
| 8 |
-
"decoder_sparse_step": 1,
|
| 9 |
-
"dtype": "bfloat16",
|
| 10 |
-
"eos_token_id": 151645,
|
| 11 |
-
"head_dim": 256,
|
| 12 |
-
"hidden_act": "silu",
|
| 13 |
-
"hidden_size": 2048,
|
| 14 |
-
"initializer_range": 0.02,
|
| 15 |
-
"intermediate_size": 5120,
|
| 16 |
-
"layer_types": [
|
| 17 |
-
"linear_attention",
|
| 18 |
-
"linear_attention",
|
| 19 |
-
"linear_attention",
|
| 20 |
-
"full_attention",
|
| 21 |
-
"linear_attention",
|
| 22 |
-
"linear_attention",
|
| 23 |
-
"linear_attention",
|
| 24 |
-
"full_attention",
|
| 25 |
-
"linear_attention",
|
| 26 |
-
"linear_attention",
|
| 27 |
-
"linear_attention",
|
| 28 |
-
"full_attention",
|
| 29 |
-
"linear_attention",
|
| 30 |
-
"linear_attention",
|
| 31 |
-
"linear_attention",
|
| 32 |
-
"full_attention",
|
| 33 |
-
"linear_attention",
|
| 34 |
-
"linear_attention",
|
| 35 |
-
"linear_attention",
|
| 36 |
-
"full_attention",
|
| 37 |
-
"linear_attention",
|
| 38 |
-
"linear_attention",
|
| 39 |
-
"linear_attention",
|
| 40 |
-
"full_attention",
|
| 41 |
-
"linear_attention",
|
| 42 |
-
"linear_attention",
|
| 43 |
-
"linear_attention",
|
| 44 |
-
"full_attention",
|
| 45 |
-
"linear_attention",
|
| 46 |
-
"linear_attention",
|
| 47 |
-
"linear_attention",
|
| 48 |
-
"full_attention",
|
| 49 |
-
"linear_attention",
|
| 50 |
-
"linear_attention",
|
| 51 |
-
"linear_attention",
|
| 52 |
-
"full_attention",
|
| 53 |
-
"linear_attention",
|
| 54 |
-
"linear_attention",
|
| 55 |
-
"linear_attention",
|
| 56 |
-
"full_attention",
|
| 57 |
-
"linear_attention",
|
| 58 |
-
"linear_attention",
|
| 59 |
-
"linear_attention",
|
| 60 |
-
"full_attention",
|
| 61 |
-
"linear_attention",
|
| 62 |
-
"linear_attention",
|
| 63 |
-
"linear_attention",
|
| 64 |
-
"full_attention"
|
| 65 |
-
],
|
| 66 |
-
"linear_conv_kernel_dim": 4,
|
| 67 |
-
"linear_key_head_dim": 128,
|
| 68 |
-
"linear_num_key_heads": 16,
|
| 69 |
-
"linear_num_value_heads": 32,
|
| 70 |
-
"linear_value_head_dim": 128,
|
| 71 |
-
"max_position_embeddings": 262144,
|
| 72 |
-
"merge_args": {
|
| 73 |
-
"batch_size": 2,
|
| 74 |
-
"cache_dir": null,
|
| 75 |
-
"dataset": "c4+math+code",
|
| 76 |
-
"download": false,
|
| 77 |
-
"group_size": 16,
|
| 78 |
-
"grouping": "ream",
|
| 79 |
-
"merge_size": 256,
|
| 80 |
-
"merging": "logits+weights",
|
| 81 |
-
"mix_ratio": "0.0,0.3,0.7",
|
| 82 |
-
"model": "/root/.cache/huggingface/hub/models--samuelcardillo--Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled/snapshots/1a298f7f385aa744169db630a16d28e56b5566ff",
|
| 83 |
-
"mtp_safe_tensors": null,
|
| 84 |
-
"no_gate_output": false,
|
| 85 |
-
"no_gated_sim": false,
|
| 86 |
-
"no_sequential": false,
|
| 87 |
-
"saliency": "reap",
|
| 88 |
-
"save_path": "./samuel-ream-256"
|
| 89 |
-
},
|
| 90 |
-
"mlp_only_layers": [],
|
| 91 |
-
"model_type": "qwen3_next",
|
| 92 |
-
"moe_intermediate_size": 512,
|
| 93 |
-
"norm_topk_prob": true,
|
| 94 |
-
"num_attention_heads": 16,
|
| 95 |
-
"num_experts": 256,
|
| 96 |
-
"num_experts_per_tok": 10,
|
| 97 |
-
"num_hidden_layers": 48,
|
| 98 |
-
"num_key_value_heads": 2,
|
| 99 |
-
"output_router_logits": false,
|
| 100 |
-
"pad_token_id": 151643,
|
| 101 |
-
"partial_rotary_factor": 0.25,
|
| 102 |
-
"rms_norm_eps": 1e-06,
|
| 103 |
-
"rope_parameters": {
|
| 104 |
-
"partial_rotary_factor": 0.25,
|
| 105 |
-
"rope_theta": 5000000,
|
| 106 |
-
"rope_type": "default"
|
| 107 |
-
},
|
| 108 |
-
"router_aux_loss_coef": 0.001,
|
| 109 |
-
"shared_expert_intermediate_size": 512,
|
| 110 |
-
"tie_word_embeddings": false,
|
| 111 |
-
"transformers_version": "5.6.0",
|
| 112 |
-
"use_cache": false,
|
| 113 |
-
"use_sliding_window": false,
|
| 114 |
-
"vocab_size": 151936
|
| 115 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0957b0786824a59122d4a044977d80ea79125e3b9c1606c373a3e9c110efa687
|
| 3 |
-
size 3425524216
|
|
|
|
|
|
|
|
|
|
|
|
blobs/17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:17e8663108143edef179a090f8d51262160da458e2f67ecc849fdb6103de89ae
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/1ac848e36c5e24a7f31f64b5631eb8890ffe916f
DELETED
|
@@ -1,117 +0,0 @@
|
|
| 1 |
-
{% macro render_extra_keys(json_dict, handled_keys) %}
|
| 2 |
-
{%- if json_dict is mapping %}
|
| 3 |
-
{%- for json_key in json_dict if json_key not in handled_keys %}
|
| 4 |
-
{%- if json_dict[json_key] is string %}
|
| 5 |
-
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
| 6 |
-
{%- else %}
|
| 7 |
-
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
| 8 |
-
{%- endif %}
|
| 9 |
-
{%- endfor %}
|
| 10 |
-
{%- endif %}
|
| 11 |
-
{%- endmacro %}
|
| 12 |
-
|
| 13 |
-
{%- if messages[0]["role"] == "system" %}
|
| 14 |
-
{%- set system_message = messages[0]["content"] %}
|
| 15 |
-
{%- set loop_messages = messages[1:] %}
|
| 16 |
-
{%- else %}
|
| 17 |
-
{%- set loop_messages = messages %}
|
| 18 |
-
{%- endif %}
|
| 19 |
-
|
| 20 |
-
{%- if not tools is defined %}
|
| 21 |
-
{%- set tools = [] %}
|
| 22 |
-
{%- endif %}
|
| 23 |
-
|
| 24 |
-
{%- if system_message is defined %}
|
| 25 |
-
{{- "<|im_start|>system\n" + system_message }}
|
| 26 |
-
{%- else %}
|
| 27 |
-
{%- if tools is iterable and tools | length > 0 %}
|
| 28 |
-
{{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
|
| 29 |
-
{%- endif %}
|
| 30 |
-
{%- endif %}
|
| 31 |
-
{%- if tools is iterable and tools | length > 0 %}
|
| 32 |
-
{{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
|
| 33 |
-
{{- "<tools>" }}
|
| 34 |
-
{%- for tool in tools %}
|
| 35 |
-
{%- if tool.function is defined %}
|
| 36 |
-
{%- set tool = tool.function %}
|
| 37 |
-
{%- endif %}
|
| 38 |
-
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
| 39 |
-
{%- if tool.description is defined %}
|
| 40 |
-
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
| 41 |
-
{%- endif %}
|
| 42 |
-
{{- '\n<parameters>' }}
|
| 43 |
-
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
| 44 |
-
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 45 |
-
{{- '\n<parameter>' }}
|
| 46 |
-
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
| 47 |
-
{%- if param_fields.type is defined %}
|
| 48 |
-
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
| 49 |
-
{%- endif %}
|
| 50 |
-
{%- if param_fields.description is defined %}
|
| 51 |
-
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
| 52 |
-
{%- endif %}
|
| 53 |
-
{%- set handled_keys = ['name', 'type', 'description'] %}
|
| 54 |
-
{{- render_extra_keys(param_fields, handled_keys) }}
|
| 55 |
-
{{- '\n</parameter>' }}
|
| 56 |
-
{%- endfor %}
|
| 57 |
-
{%- endif %}
|
| 58 |
-
{%- set handled_keys = ['type', 'properties'] %}
|
| 59 |
-
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
| 60 |
-
{{- '\n</parameters>' }}
|
| 61 |
-
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
| 62 |
-
{{- render_extra_keys(tool, handled_keys) }}
|
| 63 |
-
{{- '\n</function>' }}
|
| 64 |
-
{%- endfor %}
|
| 65 |
-
{{- "\n</tools>" }}
|
| 66 |
-
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 67 |
-
{%- endif %}
|
| 68 |
-
{%- if system_message is defined %}
|
| 69 |
-
{{- '<|im_end|>\n' }}
|
| 70 |
-
{%- else %}
|
| 71 |
-
{%- if tools is iterable and tools | length > 0 %}
|
| 72 |
-
{{- '<|im_end|>\n' }}
|
| 73 |
-
{%- endif %}
|
| 74 |
-
{%- endif %}
|
| 75 |
-
{%- for message in loop_messages %}
|
| 76 |
-
{%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
| 77 |
-
{{- '<|im_start|>' + message.role }}
|
| 78 |
-
{%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
|
| 79 |
-
{{- '\n' + message.content | trim + '\n' }}
|
| 80 |
-
{%- endif %}
|
| 81 |
-
{%- for tool_call in message.tool_calls %}
|
| 82 |
-
{%- if tool_call.function is defined %}
|
| 83 |
-
{%- set tool_call = tool_call.function %}
|
| 84 |
-
{%- endif %}
|
| 85 |
-
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 86 |
-
{%- if tool_call.arguments is defined %}
|
| 87 |
-
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 88 |
-
{{- '<parameter=' + args_name + '>\n' }}
|
| 89 |
-
{%- set args_value = args_value if args_value is string else args_value | tojson | safe %}
|
| 90 |
-
{{- args_value }}
|
| 91 |
-
{{- '\n</parameter>\n' }}
|
| 92 |
-
{%- endfor %}
|
| 93 |
-
{%- endif %}
|
| 94 |
-
{{- '</function>\n</tool_call>' }}
|
| 95 |
-
{%- endfor %}
|
| 96 |
-
{{- '<|im_end|>\n' }}
|
| 97 |
-
{%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
|
| 98 |
-
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
| 99 |
-
{%- elif message.role == "tool" %}
|
| 100 |
-
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 101 |
-
{{- '<|im_start|>user' }}
|
| 102 |
-
{%- endif %}
|
| 103 |
-
{{- '\n<tool_response>\n' }}
|
| 104 |
-
{{- message.content }}
|
| 105 |
-
{{- '\n</tool_response>' }}
|
| 106 |
-
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 107 |
-
{{- '<|im_end|>\n' }}
|
| 108 |
-
{%- elif loop.last %}
|
| 109 |
-
{{- '<|im_end|>\n' }}
|
| 110 |
-
{%- endif %}
|
| 111 |
-
{%- else %}
|
| 112 |
-
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
| 113 |
-
{%- endif %}
|
| 114 |
-
{%- endfor %}
|
| 115 |
-
{%- if add_generation_prompt %}
|
| 116 |
-
{{- '<|im_start|>assistant\n' }}
|
| 117 |
-
{%- endif %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/20ef74aaea93d005bad8ce4e7aecd870b4c85c60
DELETED
|
@@ -1,115 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"architectures": [
|
| 3 |
-
"Qwen3NextForCausalLM"
|
| 4 |
-
],
|
| 5 |
-
"attention_bias": false,
|
| 6 |
-
"attention_dropout": 0,
|
| 7 |
-
"bos_token_id": null,
|
| 8 |
-
"decoder_sparse_step": 1,
|
| 9 |
-
"dtype": "bfloat16",
|
| 10 |
-
"eos_token_id": 151645,
|
| 11 |
-
"head_dim": 256,
|
| 12 |
-
"hidden_act": "silu",
|
| 13 |
-
"hidden_size": 2048,
|
| 14 |
-
"initializer_range": 0.02,
|
| 15 |
-
"intermediate_size": 5120,
|
| 16 |
-
"layer_types": [
|
| 17 |
-
"linear_attention",
|
| 18 |
-
"linear_attention",
|
| 19 |
-
"linear_attention",
|
| 20 |
-
"full_attention",
|
| 21 |
-
"linear_attention",
|
| 22 |
-
"linear_attention",
|
| 23 |
-
"linear_attention",
|
| 24 |
-
"full_attention",
|
| 25 |
-
"linear_attention",
|
| 26 |
-
"linear_attention",
|
| 27 |
-
"linear_attention",
|
| 28 |
-
"full_attention",
|
| 29 |
-
"linear_attention",
|
| 30 |
-
"linear_attention",
|
| 31 |
-
"linear_attention",
|
| 32 |
-
"full_attention",
|
| 33 |
-
"linear_attention",
|
| 34 |
-
"linear_attention",
|
| 35 |
-
"linear_attention",
|
| 36 |
-
"full_attention",
|
| 37 |
-
"linear_attention",
|
| 38 |
-
"linear_attention",
|
| 39 |
-
"linear_attention",
|
| 40 |
-
"full_attention",
|
| 41 |
-
"linear_attention",
|
| 42 |
-
"linear_attention",
|
| 43 |
-
"linear_attention",
|
| 44 |
-
"full_attention",
|
| 45 |
-
"linear_attention",
|
| 46 |
-
"linear_attention",
|
| 47 |
-
"linear_attention",
|
| 48 |
-
"full_attention",
|
| 49 |
-
"linear_attention",
|
| 50 |
-
"linear_attention",
|
| 51 |
-
"linear_attention",
|
| 52 |
-
"full_attention",
|
| 53 |
-
"linear_attention",
|
| 54 |
-
"linear_attention",
|
| 55 |
-
"linear_attention",
|
| 56 |
-
"full_attention",
|
| 57 |
-
"linear_attention",
|
| 58 |
-
"linear_attention",
|
| 59 |
-
"linear_attention",
|
| 60 |
-
"full_attention",
|
| 61 |
-
"linear_attention",
|
| 62 |
-
"linear_attention",
|
| 63 |
-
"linear_attention",
|
| 64 |
-
"full_attention"
|
| 65 |
-
],
|
| 66 |
-
"linear_conv_kernel_dim": 4,
|
| 67 |
-
"linear_key_head_dim": 128,
|
| 68 |
-
"linear_num_key_heads": 16,
|
| 69 |
-
"linear_num_value_heads": 32,
|
| 70 |
-
"linear_value_head_dim": 128,
|
| 71 |
-
"max_position_embeddings": 32768,
|
| 72 |
-
"merge_args": {
|
| 73 |
-
"batch_size": 2,
|
| 74 |
-
"cache_dir": null,
|
| 75 |
-
"dataset": "c4+math+code",
|
| 76 |
-
"download": false,
|
| 77 |
-
"group_size": 16,
|
| 78 |
-
"grouping": "ream",
|
| 79 |
-
"merge_size": 256,
|
| 80 |
-
"merging": "logits+weights",
|
| 81 |
-
"mix_ratio": "0.0,0.3,0.7",
|
| 82 |
-
"model": "/root/.cache/huggingface/hub/models--samuelcardillo--Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled/snapshots/1a298f7f385aa744169db630a16d28e56b5566ff",
|
| 83 |
-
"mtp_safe_tensors": null,
|
| 84 |
-
"no_gate_output": false,
|
| 85 |
-
"no_gated_sim": false,
|
| 86 |
-
"no_sequential": false,
|
| 87 |
-
"saliency": "reap",
|
| 88 |
-
"save_path": "./samuel-ream-256"
|
| 89 |
-
},
|
| 90 |
-
"mlp_only_layers": [],
|
| 91 |
-
"model_type": "qwen3_next",
|
| 92 |
-
"moe_intermediate_size": 512,
|
| 93 |
-
"norm_topk_prob": true,
|
| 94 |
-
"num_attention_heads": 16,
|
| 95 |
-
"num_experts": 256,
|
| 96 |
-
"num_experts_per_tok": 10,
|
| 97 |
-
"num_hidden_layers": 48,
|
| 98 |
-
"num_key_value_heads": 2,
|
| 99 |
-
"output_router_logits": false,
|
| 100 |
-
"pad_token_id": 151643,
|
| 101 |
-
"partial_rotary_factor": 0.25,
|
| 102 |
-
"rms_norm_eps": 1e-06,
|
| 103 |
-
"rope_parameters": {
|
| 104 |
-
"partial_rotary_factor": 0.25,
|
| 105 |
-
"rope_theta": 5000000,
|
| 106 |
-
"rope_type": "default"
|
| 107 |
-
},
|
| 108 |
-
"router_aux_loss_coef": 0.001,
|
| 109 |
-
"shared_expert_intermediate_size": 512,
|
| 110 |
-
"tie_word_embeddings": false,
|
| 111 |
-
"transformers_version": "5.6.0",
|
| 112 |
-
"use_cache": false,
|
| 113 |
-
"use_sliding_window": false,
|
| 114 |
-
"vocab_size": 151936
|
| 115 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:211cc1d4fc03b7de0041bc9e194d7d889c2cb00fce5c9915d2c4c77be3880b71
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:2ab88e298a5b1dd02478827fd4bd5816eeb392e6b05be84512d5ad2eceefb421
|
| 3 |
-
size 3534503480
|
|
|
|
|
|
|
|
|
|
|
|
blobs/39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:39430ec1eb3486aecaa978d4334cf0be37fceaf423a0ce1ded1a8ce883d6cdf8
|
| 3 |
-
size 3303559928
|
|
|
|
|
|
|
|
|
|
|
|
blobs/395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:395037974254925097d090780270f6d169c87d1f3734cb475678c77e7f8972ba
|
| 3 |
-
size 3425524216
|
|
|
|
|
|
|
|
|
|
|
|
blobs/52373fe24473b1aa44333d318f578ae6bf04b49b
DELETED
|
@@ -1,36 +0,0 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:556de3b29b54a32a42f7fa4af4f34de64bf80b8eebaf4c8c1d2f52c42aba0829
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:56f3287698d2d004c220487a786e2b683f9d5c73bb805b93bcacab9d160acd5f
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/5d5e4a4c418e93217947ca434c046d36ff634400
DELETED
|
@@ -1,12 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"do_sample": true,
|
| 3 |
-
"eos_token_id": [
|
| 4 |
-
151645,
|
| 5 |
-
151643
|
| 6 |
-
],
|
| 7 |
-
"pad_token_id": 151643,
|
| 8 |
-
"temperature": 1.0,
|
| 9 |
-
"top_k": 40,
|
| 10 |
-
"top_p": 0.95,
|
| 11 |
-
"transformers_version": "5.6.0"
|
| 12 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/60e155923df59cf410facf6dce3ddb14fac6b37d
DELETED
|
@@ -1,31 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"add_prefix_space": false,
|
| 3 |
-
"backend": "tokenizers",
|
| 4 |
-
"bos_token": null,
|
| 5 |
-
"clean_up_tokenization_spaces": false,
|
| 6 |
-
"eos_token": "<|im_end|>",
|
| 7 |
-
"errors": "replace",
|
| 8 |
-
"extra_special_tokens": [
|
| 9 |
-
"<|im_start|>",
|
| 10 |
-
"<|im_end|>",
|
| 11 |
-
"<|object_ref_start|>",
|
| 12 |
-
"<|object_ref_end|>",
|
| 13 |
-
"<|box_start|>",
|
| 14 |
-
"<|box_end|>",
|
| 15 |
-
"<|quad_start|>",
|
| 16 |
-
"<|quad_end|>",
|
| 17 |
-
"<|vision_start|>",
|
| 18 |
-
"<|vision_end|>",
|
| 19 |
-
"<|vision_pad|>",
|
| 20 |
-
"<|image_pad|>",
|
| 21 |
-
"<|video_pad|>"
|
| 22 |
-
],
|
| 23 |
-
"is_local": true,
|
| 24 |
-
"local_files_only": false,
|
| 25 |
-
"model_max_length": 32768,
|
| 26 |
-
"pad_token": "<|im_end|>",
|
| 27 |
-
"split_special_tokens": false,
|
| 28 |
-
"tokenizer_class": "Qwen2Tokenizer",
|
| 29 |
-
"unk_token": null,
|
| 30 |
-
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
|
| 31 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:69b0b3e53bedd66d3561300c1208915562560b944b0584e604f2a02eb1be3a4d
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:6fb5182cb91d04ddc9fd3906b67134716abb8996c97a5f69c7d551879ec2d057
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:71789257ab50843b7fa2bfa296986db90c04d68e1416c8420561fb950ea8a0fa
|
| 3 |
-
size 3303558816
|
|
|
|
|
|
|
|
|
|
|
|
blobs/83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:83741f1c53a951fea319eff340d1d3fca27ed7b28f9675304024e77b430da47c
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:90935ab97b66885a0eb8a8f0270edcbb0bc0f572f4c6a0855f1ad46fac71b411
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:977e094c01e07d0a662e96808a4825d2f84d3dd69f6e454b43792859a8afe2dd
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:9a1e717c7a7119026204060cfcf52ecd01fb56ea1c00680ec242d016e3d6ccd0
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:a0aad645b369fc3236fded010e45116b7d8356c16c223b075b40ad7c29902a74
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:adc0c122c7bd42c5edb2e10c31f460399e71d53e77abe3e54eb3067622862b0e
|
| 3 |
-
size 1135685400
|
|
|
|
|
|
|
|
|
|
|
|
blobs/af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:af9b55bbf0ff6acd64496d93f586466fe11ca96fc5dd3c5b3a5d207af3bc8d56
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:b2f80e9f333bf7d1c65bc70960047a739aa57c2ecf7311ebe0165f107cac6620
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:b409eb83ecc2300ed10890938ac5f637f45bfbedb8dfd97f896847658a3967ee
|
| 3 |
-
size 3303558816
|
|
|
|
|
|
|
|
|
|
|
|
blobs/bbfbbce7c8eaa0684336e3b64ce4a02bacdd6685
DELETED
|
@@ -1,110 +0,0 @@
|
|
| 1 |
-
---
|
| 2 |
-
license: apache-2.0
|
| 3 |
-
language:
|
| 4 |
-
- en
|
| 5 |
-
base_model:
|
| 6 |
-
- samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled
|
| 7 |
-
pipeline_tag: text-generation
|
| 8 |
-
library_name: transformers
|
| 9 |
-
tags:
|
| 10 |
-
- qwen3next
|
| 11 |
-
- Code
|
| 12 |
-
- REAM
|
| 13 |
-
- custom-merge
|
| 14 |
-
---
|
| 15 |
-
|
| 16 |
-
# Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B
|
| 17 |
-
|
| 18 |
-
A REAM-merged version of [samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled](https://huggingface.co/samuelcardillo/Qwen3-Coder-Next-Opus-4.6-Reasoning-Distilled), reducing 512 experts to 256 (50% reduction) using [REAM (Router-weighted Expert Activation Merging)](https://github.com/SamsungSAILMontreal/ream).
|
| 19 |
-
|
| 20 |
-
## Status: Experimental
|
| 21 |
-
|
| 22 |
-
To my knowledge this is the **first publicly released REAM model at 50% expert reduction on the `qwen3_next` architecture**. Prior published REAM Coder-Next models (e.g. [bknyaz/Qwen3-Coder-Next-REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM)) target 25% reduction, which the REAM paper authors recommend for production use.
|
| 23 |
-
|
| 24 |
-
**The REAM authors explicitly note that merging quality degrades at 50% compared to 25%, and the REAP authors empirically demonstrate that pruning (REAP) outperforms merging (REAM) at 50% reduction on generative tasks.** This release is an experimental artifact rather than a recommended production model. Comprehensive evaluation has not been run.
|
| 25 |
-
|
| 26 |
-
## Model Details
|
| 27 |
-
|
| 28 |
-
- **Architecture:** `qwen3_next` (hybrid Gated DeltaNet + Attention + MoE)
|
| 29 |
-
- **Parameters:** ~40B total, 3B active per forward pass
|
| 30 |
-
- **Layers:** 48
|
| 31 |
-
- **Experts:** 256 (merged from 512)
|
| 32 |
-
- **Hidden dim:** 2048
|
| 33 |
-
- **moe_intermediate_size:** 512
|
| 34 |
-
- **Precision:** bf16
|
| 35 |
-
- **Source:** Samuel Cardillo’s full fine-tune of Qwen3-Coder-Next on Claude Opus 4.6 reasoning distillation data
|
| 36 |
-
|
| 37 |
-
## REAM Configuration Used
|
| 38 |
-
|
| 39 |
-
```
|
| 40 |
-
saliency: reap
|
| 41 |
-
merging: logits+weights
|
| 42 |
-
grouping: ream
|
| 43 |
-
merge_size: 256 (50% of original 512)
|
| 44 |
-
group_size: 16
|
| 45 |
-
calibration: c4+math+code (default mix 0.0:0.3:0.7)
|
| 46 |
-
batch_size: 2
|
| 47 |
-
sequential: True
|
| 48 |
-
```
|
| 49 |
-
|
| 50 |
-
Per-layer reduction confirmed cleanly during the run: each layer’s 512 experts merged to 256 with the expected 2:1 parameter reduction (e.g. 1.61B → 808M expert params per layer).
|
| 51 |
-
|
| 52 |
-
## Intended Use
|
| 53 |
-
|
| 54 |
-
- Research into expert merging methods on hybrid MoE architectures
|
| 55 |
-
- Comparison baseline for staged-REAM or other quality-preservation techniques at high compression ratios
|
| 56 |
-
- Base for further fine-tuning experiments where 40B-class size is needed
|
| 57 |
-
|
| 58 |
-
## Not Recommended For
|
| 59 |
-
|
| 60 |
-
- Production deployment without further evaluation. Use [bknyaz’s 25% REAM](https://huggingface.co/bknyaz/Qwen3-Coder-Next-REAM) or [lovedheart’s REAP-40B](https://huggingface.co/lovedheart/Qwen3-Coder-Next-REAP-40B-A3B) instead.
|
| 61 |
-
- Code generation at production quality without verification on your specific task.
|
| 62 |
-
- Comparison against the original Samuel 80B without acknowledging the 50% capacity reduction.
|
| 63 |
-
|
| 64 |
-
## Limitations
|
| 65 |
-
|
| 66 |
-
- 50% expert reduction necessarily loses capability compared to the 512-expert source.
|
| 67 |
-
- REAM at 50% is documented as suboptimal compared to REAP at 50% for generative tasks (Cerebras 2025).
|
| 68 |
-
- No benchmarks have been run on this specific checkpoint.
|
| 69 |
-
- Inherits all biases and limitations from Samuel’s source model and the original Qwen3-Coder-Next pretraining.
|
| 70 |
-
|
| 71 |
-
## Loading
|
| 72 |
-
|
| 73 |
-
```python
|
| 74 |
-
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 75 |
-
|
| 76 |
-
model = AutoModelForCausalLM.from_pretrained(
|
| 77 |
-
"Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
|
| 78 |
-
torch_dtype="auto",
|
| 79 |
-
device_map="auto",
|
| 80 |
-
trust_remote_code=True,
|
| 81 |
-
)
|
| 82 |
-
tokenizer = AutoTokenizer.from_pretrained(
|
| 83 |
-
"Johnblick187/Qwen3-Coder-Next-Claude-Opus-4.6-Reasoning-Distilled-REAM-40B-A3B",
|
| 84 |
-
trust_remote_code=True,
|
| 85 |
-
)
|
| 86 |
-
```
|
| 87 |
-
|
| 88 |
-
~80GB VRAM needed in bf16, ~20GB at 4-bit.
|
| 89 |
-
|
| 90 |
-
## Acknowledgements
|
| 91 |
-
|
| 92 |
-
- [Samuel Cardillo](https://huggingface.co/samuelcardillo) for the source fine-tune
|
| 93 |
-
- [Samsung SAIL Montreal](https://github.com/SamsungSAILMontreal/ream) and [Boris Knyazev](https://bknyaz.github.io/blog/2026/moe/) for REAM
|
| 94 |
-
- [Cerebras Research](https://github.com/CerebrasResearch/reap) for REAP and the comparative analysis
|
| 95 |
-
- [Qwen team](https://huggingface.co/Qwen) for Qwen3-Coder-Next
|
| 96 |
-
- [lovedheart](https://huggingface.co/lovedheart) for prior REAP work on this architecture
|
| 97 |
-
|
| 98 |
-
## Training Compute
|
| 99 |
-
|
| 100 |
-
Single H200 NVL on RunPod. ~250GB system RAM used during merge (model loaded `device_map='cpu'` per REAM script default). ~3.5 hours runtime. Sequential layer-by-layer merge mode.
|
| 101 |
-
|
| 102 |
-
## References
|
| 103 |
-
|
| 104 |
-
- REAM paper: https://arxiv.org/abs/2604.04356
|
| 105 |
-
- REAP paper: https://arxiv.org/abs/2510.13999
|
| 106 |
-
- bknyaz blog: https://bknyaz.github.io/blog/2026/moe/
|
| 107 |
-
|
| 108 |
-
## Disclaimer
|
| 109 |
-
|
| 110 |
-
The “Claude Opus 4.6” attribution in the name is propagated from Samuel’s source model name. This model and its source were not produced by, endorsed by, or affiliated with Anthropic. Use of this model should comply with Anthropic’s terms regarding training on Claude outputs if applicable to your use case.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:bddfba269d75c85c576395ec2aa3f1a1f33008f2842bbaaaecf0a8b5b34f0685
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
|
| 3 |
-
size 11422650
|
|
|
|
|
|
|
|
|
|
|
|
blobs/c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:c1d3ff95298947c49a4582ae9c8c3193a3a8806aabc64b3b717dc56328f433c8
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|
blobs/c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:c4e2d125184b33e07b4e5cd6fee165a0dbc9f250b04715d05c19c8a1c54a8639
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/dc13e5abfbc7bb1f8ca8e8c564cd2d1b10d1a273
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
blobs/e6f3a82c69f4735fd65b222bc6af006d4bd23dcd
DELETED
|
@@ -1,30 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"add_prefix_space": false,
|
| 3 |
-
"backend": "tokenizers",
|
| 4 |
-
"bos_token": null,
|
| 5 |
-
"clean_up_tokenization_spaces": false,
|
| 6 |
-
"eos_token": "<|im_end|>",
|
| 7 |
-
"errors": "replace",
|
| 8 |
-
"extra_special_tokens": [
|
| 9 |
-
"<|im_start|>",
|
| 10 |
-
"<|im_end|>",
|
| 11 |
-
"<|object_ref_start|>",
|
| 12 |
-
"<|object_ref_end|>",
|
| 13 |
-
"<|box_start|>",
|
| 14 |
-
"<|box_end|>",
|
| 15 |
-
"<|quad_start|>",
|
| 16 |
-
"<|quad_end|>",
|
| 17 |
-
"<|vision_start|>",
|
| 18 |
-
"<|vision_end|>",
|
| 19 |
-
"<|vision_pad|>",
|
| 20 |
-
"<|image_pad|>",
|
| 21 |
-
"<|video_pad|>"
|
| 22 |
-
],
|
| 23 |
-
"is_local": true,
|
| 24 |
-
"local_files_only": false,
|
| 25 |
-
"model_max_length": 1048576,
|
| 26 |
-
"pad_token": "<|endoftext|>",
|
| 27 |
-
"split_special_tokens": false,
|
| 28 |
-
"tokenizer_class": "Qwen2Tokenizer",
|
| 29 |
-
"unk_token": null
|
| 30 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
blobs/f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:f0a3f1ff7cfa313d65a1085e7dfdbc2002df86b98043dfd938aa617518f79237
|
| 3 |
-
size 3425525792
|
|
|
|
|
|
|
|
|
|
|
|
blobs/fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:fe5e1594ada6579b294c7e20589d83c60a7b16f1031fa595cba8d49edf4611ce
|
| 3 |
-
size 3303560376
|
|
|
|
|
|
|
|
|
|
|
|