gev-e4b-mev / dense.json
hilman2's picture
Upload gev-e4b-mev
d803596 verified
Raw History Blame Contribute Delete
8.22 kB
{
"layers": 42,
"hidden": 2560,
"ple": 256,
"heads": 8,
"window": 512,
"layer_types": [
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention"
],
"head_dim": [
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512,
256,
256,
256,
256,
256,
512
],
"kv_heads": [
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2,
2
],
"kv_source": {
"24": 22,
"25": 22,
"26": 22,
"27": 22,
"28": 22,
"29": 23,
"30": 22,
"31": 22,
"32": 22,
"33": 22,
"34": 22,
"35": 23,
"36": 22,
"37": 22,
"38": 22,
"39": 22,
"40": 22,
"41": 23
},
"sources": [
22,
23
],
"inv_freq": {
"256": [
1.0,
0.9305720329284668,
0.8659643530845642,
0.8058422207832336,
0.7498942017555237,
0.6978305578231812,
0.6493816375732422,
0.6042963862419128,
0.5623413324356079,
0.5232991576194763,
0.486967533826828,
0.45315834879875183,
0.4216965138912201,
0.3924189805984497,
0.36517414450645447,
0.33982083201408386,
0.3162277638912201,
0.2942727208137512,
0.2738419771194458,
0.25482967495918274,
0.23713736236095428,
0.22067339718341827,
0.20535250008106232,
0.1910952925682068,
0.17782793939113617,
0.16548170149326324,
0.1539926528930664,
0.14330126345157623,
0.1333521455526352,
0.12409377098083496,
0.11547820270061493,
0.10746078193187714,
0.10000000149011612,
0.09305720031261444,
0.08659642934799194,
0.08058421313762665,
0.07498941570520401,
0.069783054292202,
0.06493816524744034,
0.0604296438395977,
0.05623412877321243,
0.05232991278171539,
0.04869675263762474,
0.04531583562493324,
0.04216964915394783,
0.03924189880490303,
0.03651741147041321,
0.033982083201408386,
0.03162277862429619,
0.02942727319896221,
0.0273841954767704,
0.025482967495918274,
0.0237137358635664,
0.022067340090870857,
0.02053525112569332,
0.019109530374407768,
0.017782794311642647,
0.016548171639442444,
0.015399265103042126,
0.014330125413835049,
0.01333521492779255,
0.012409376911818981,
0.011547819711267948,
0.010746078565716743,
0.009999999776482582,
0.009305720217525959,
0.00865964312106371,
0.00805842224508524,
0.007498942315578461,
0.006978305988013744,
0.006493816617876291,
0.006042964290827513,
0.005623413249850273,
0.005232991185039282,
0.004869675263762474,
0.004531583748757839,
0.0042169648222625256,
0.003924189601093531,
0.003651741426438093,
0.003398208413273096,
0.003162277862429619,
0.002942727180197835,
0.0027384196873754263,
0.0025482967030256987,
0.0023713738191872835,
0.002206734148785472,
0.0020535250660032034,
0.001910952851176262,
0.0017782794311642647,
0.00165481714066118,
0.0015399265103042126,
0.0014330126577988267,
0.0013335214462131262,
0.0012409378541633487,
0.0011547820176929235,
0.00107460783328861,
0.0010000000474974513,
0.0009305721032433212,
0.0008659643353894353,
0.0008058422245085239,
0.0007498941849917173,
0.0006978305755183101,
0.0006493816617876291,
0.0006042963941581547,
0.000562341301701963,
0.0005232990952208638,
0.00048696750309318304,
0.00045315833995118737,
0.00042169648804701865,
0.0003924189950339496,
0.00036517411353997886,
0.00033982080640271306,
0.0003162277862429619,
0.0002942727296613157,
0.00027384195709601045,
0.00025482967612333596,
0.0002371373848291114,
0.00022067342069931328,
0.00020535249495878816,
0.00019109529966954142,
0.00017782794020604342,
0.00016548171697650105,
0.00015399264520965517,
0.00014330126577988267,
0.0001333521504420787,
0.00012409377086441964,
0.00011547819303814322,
0.00010746077896328643
],
"512": [
1.0,
0.9474635124206543,
0.8976871371269226,
0.8505258560180664,
0.8058422207832336,
0.7635061144828796,
0.7233941555023193,
0.6853895783424377,
0.6493816375732422,
0.615265429019928,
0.5829415321350098,
0.5523158311843872,
0.5232991576194763,
0.49580681324005127,
0.46975886821746826,
0.44507938623428345,
0.4216965138912201,
0.39954206347465515,
0.37855154275894165,
0.35866376757621765,
0.33982083201408386,
0.32196784019470215,
0.305052787065506,
0.2890264093875885,
0.2738419771194458,
0.25945526361465454,
0.24582439661026,
0.2329096496105194,
0.22067339718341827,
0.20907999575138092,
0.1980956792831421,
0.1876884251832962,
0.17782793939113617,
0.16848549246788025,
0.15963385999202728,
0.15124726295471191,
0.14330126345157623,
0.1357727199792862,
0.12863969802856445,
0.12188142538070679,
0.11547820270061493,
0.10941138118505478,
0.10366329550743103,
0.0982171818614006,
0.09305720031261444,
0.08816830813884735,
0.08353625982999802,
0.07914755493402481,
0.07498941570520401,
0.07104974240064621,
0.06731703877449036,
0.06378044188022614,
0.0604296438395977,
0.05725487694144249,
0.05424690619111061,
0.051396969705820084,
0.04869675263762474,
0.046138398349285126,
0.04371444880962372,
0.04141784459352493,
0.03924189880490303,
0.03718026727437973,
0.03522694483399391,
0.03337624669075012,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"sizes": [
512,
576,
640,
704,
768,
832,
896,
960,
1024
],
"gpu_from": 14,
"cpu_layers": 2,
"cpu_rows": true
}