{ "layers": 42, "hidden": 2560, "ple": 256, "heads": 8, "window": 512, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "head_dim": [ 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512, 256, 256, 256, 256, 256, 512 ], "kv_heads": [ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ], "kv_source": { "24": 22, "25": 22, "26": 22, "27": 22, "28": 22, "29": 23, "30": 22, "31": 22, "32": 22, "33": 22, "34": 22, "35": 23, "36": 22, "37": 22, "38": 22, "39": 22, "40": 22, "41": 23 }, "sources": [ 22, 23 ], "inv_freq": { "256": [ 1.0, 0.9305720329284668, 0.8659643530845642, 0.8058422207832336, 0.7498942017555237, 0.6978305578231812, 0.6493816375732422, 0.6042963862419128, 0.5623413324356079, 0.5232991576194763, 0.486967533826828, 0.45315834879875183, 0.4216965138912201, 0.3924189805984497, 0.36517414450645447, 0.33982083201408386, 0.3162277638912201, 0.2942727208137512, 0.2738419771194458, 0.25482967495918274, 0.23713736236095428, 0.22067339718341827, 0.20535250008106232, 0.1910952925682068, 0.17782793939113617, 0.16548170149326324, 0.1539926528930664, 0.14330126345157623, 0.1333521455526352, 0.12409377098083496, 0.11547820270061493, 0.10746078193187714, 0.10000000149011612, 0.09305720031261444, 0.08659642934799194, 0.08058421313762665, 0.07498941570520401, 0.069783054292202, 0.06493816524744034, 0.0604296438395977, 0.05623412877321243, 0.05232991278171539, 0.04869675263762474, 0.04531583562493324, 0.04216964915394783, 0.03924189880490303, 0.03651741147041321, 0.033982083201408386, 0.03162277862429619, 0.02942727319896221, 0.0273841954767704, 0.025482967495918274, 0.0237137358635664, 0.022067340090870857, 0.02053525112569332, 0.019109530374407768, 0.017782794311642647, 0.016548171639442444, 0.015399265103042126, 0.014330125413835049, 0.01333521492779255, 0.012409376911818981, 0.011547819711267948, 0.010746078565716743, 0.009999999776482582, 0.009305720217525959, 0.00865964312106371, 0.00805842224508524, 0.007498942315578461, 0.006978305988013744, 0.006493816617876291, 0.006042964290827513, 0.005623413249850273, 0.005232991185039282, 0.004869675263762474, 0.004531583748757839, 0.0042169648222625256, 0.003924189601093531, 0.003651741426438093, 0.003398208413273096, 0.003162277862429619, 0.002942727180197835, 0.0027384196873754263, 0.0025482967030256987, 0.0023713738191872835, 0.002206734148785472, 0.0020535250660032034, 0.001910952851176262, 0.0017782794311642647, 0.00165481714066118, 0.0015399265103042126, 0.0014330126577988267, 0.0013335214462131262, 0.0012409378541633487, 0.0011547820176929235, 0.00107460783328861, 0.0010000000474974513, 0.0009305721032433212, 0.0008659643353894353, 0.0008058422245085239, 0.0007498941849917173, 0.0006978305755183101, 0.0006493816617876291, 0.0006042963941581547, 0.000562341301701963, 0.0005232990952208638, 0.00048696750309318304, 0.00045315833995118737, 0.00042169648804701865, 0.0003924189950339496, 0.00036517411353997886, 0.00033982080640271306, 0.0003162277862429619, 0.0002942727296613157, 0.00027384195709601045, 0.00025482967612333596, 0.0002371373848291114, 0.00022067342069931328, 0.00020535249495878816, 0.00019109529966954142, 0.00017782794020604342, 0.00016548171697650105, 0.00015399264520965517, 0.00014330126577988267, 0.0001333521504420787, 0.00012409377086441964, 0.00011547819303814322, 0.00010746077896328643 ], "512": [ 1.0, 0.9474635124206543, 0.8976871371269226, 0.8505258560180664, 0.8058422207832336, 0.7635061144828796, 0.7233941555023193, 0.6853895783424377, 0.6493816375732422, 0.615265429019928, 0.5829415321350098, 0.5523158311843872, 0.5232991576194763, 0.49580681324005127, 0.46975886821746826, 0.44507938623428345, 0.4216965138912201, 0.39954206347465515, 0.37855154275894165, 0.35866376757621765, 0.33982083201408386, 0.32196784019470215, 0.305052787065506, 0.2890264093875885, 0.2738419771194458, 0.25945526361465454, 0.24582439661026, 0.2329096496105194, 0.22067339718341827, 0.20907999575138092, 0.1980956792831421, 0.1876884251832962, 0.17782793939113617, 0.16848549246788025, 0.15963385999202728, 0.15124726295471191, 0.14330126345157623, 0.1357727199792862, 0.12863969802856445, 0.12188142538070679, 0.11547820270061493, 0.10941138118505478, 0.10366329550743103, 0.0982171818614006, 0.09305720031261444, 0.08816830813884735, 0.08353625982999802, 0.07914755493402481, 0.07498941570520401, 0.07104974240064621, 0.06731703877449036, 0.06378044188022614, 0.0604296438395977, 0.05725487694144249, 0.05424690619111061, 0.051396969705820084, 0.04869675263762474, 0.046138398349285126, 0.04371444880962372, 0.04141784459352493, 0.03924189880490303, 0.03718026727437973, 0.03522694483399391, 0.03337624669075012, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "sizes": [ 512, 576, 640, 704, 768, 832, 896, 960, 1024 ], "gpu_from": 14, "cpu_layers": 2, "cpu_rows": true }