File size: 2,281 Bytes
3fd1a35 3706f1c 3fd1a35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | #include "ling3/precision_policy.h"
#include <iostream>
int main()try{
using namespace ling3;
auto all=ParseW8Families("attention,shared,dense,head");
auto small=ParseW8Families("kda_out,mla_in,mla_out,shared,dense,head");
for(int layer=0;layer<24;++layer){
auto prefix="model.layers."+std::to_string(layer);
bool mla=(layer+1)%4==0;
auto input=prefix+".attention."+(mla?"qkv_gate_a":"qkvfgb");
auto output=prefix+".attention.o_proj";
if(!SelectW8(all,input)||!SelectW8(all,output)||!SelectW8(small,output)||SelectW8(small,input)!=mla)
throw std::runtime_error("attention precision selection mismatch");
if(SelectW8(all,prefix+".mlp.experts.0.gate_up"))throw std::runtime_error("routed expert was selected");
}
if(!SelectW8(small,"lm_head")||!SelectW8(small,"model.layers.0.mlp.down_proj")||
!SelectW8(small,"model.layers.1.mlp.shared_experts.gate_up")||SelectW8({},"lm_head"))
throw std::runtime_error("nonattention precision selection mismatch");
#if LING3_EXPERIMENTAL_MTP
if(W8MatrixFamily("model.layers.24.attention.qkv_gate_a")!="mla_in" ||
W8MatrixFamily("model.layers.24.attention.o_proj")!="mla_out" ||
SelectW8(all,"model.layers.24.mlp.experts.0.gate_up"))
throw std::runtime_error("MTP precision family mismatch");
#else
bool disabled_mtp_rejected=false;
try{W8MatrixFamily("model.layers.24.attention.o_proj");}
catch(const std::invalid_argument&){disabled_mtp_rejected=true;}
if(!disabled_mtp_rejected)throw std::runtime_error("disabled MTP layer accepted");
#endif
bool invalid_mtp_layer=false;
try{W8MatrixFamily("model.layers.25.attention.o_proj");}
catch(const std::invalid_argument&){invalid_mtp_layer=true;}
if(!invalid_mtp_layer)throw std::runtime_error("invalid MTP layer accepted");
for(auto bad:{"kda","head,","head,,shared"}){
bool rejected=false;try{ParseW8Families(bad);}catch(const std::invalid_argument&){rejected=true;}
if(!rejected)throw std::runtime_error("invalid selector accepted");
}
std::cout<<"PASS: layer families, all-attention compatibility, routed expert exclusion, invalid selectors\n";
}catch(const std::exception&e){std::cerr<<e.what()<<'\n';return 1;}
|