one-gpu-capacity / calibration.json
thanatchon36's picture
Publish model card
0890af4 verified
Raw History Blame Contribute Delete
38.7 kB
{ "vidur": { "max": { "a100": { "eta_bw": 0.7310774976236001, "eta_c": 0.6654064678288377, "mape": 0.057558084397660726, "n": 140, "attn_bw_frac": 0.5991907381812237 }, "a40": { "eta_bw": 0.7536243228217148, "eta_c": 0.6, "mape": 0.05581423683930286, "n": 64, "attn_bw_frac": 0.7680605528836582 }, "h100": { "eta_bw": 0.8114088587438508, "eta_c": 0.6, "mape": 0.064630345092196, "n": 64, "attn_bw_frac": 0.7094333446982612 } }, "sum": { "a100": { "eta_bw": 0.8154750554470435, "eta_c": 1.2, "mape": 0.06514696945554, "n": 140, "attn_bw_frac": 0.5991907381812237 }, "a40": { "eta_bw": 0.8202894751055775, "eta_c": 1.2, "mape": 0.045614744672416406, "n": 64, "attn_bw_frac": 0.7680605528836582 }, "h100": { "eta_bw": 0.8671763154735798, "eta_c": 1.199999999999998, "mape": 0.06913968586498931, "n": 64, "attn_bw_frac": 0.7094333446982612 } }, "smooth": { "a100": { "eta_bw": 0.7500135866949069, "eta_c": 0.6454558433375267, "mape": 0.03124893839795496, "n": 140, "attn_bw_frac": 0.5991907381812237 }, "a40": { "eta_bw": 0.7675289933621684, "eta_c": 0.496990333497917, "mape": 0.04068866140227775, "n": 64, "attn_bw_frac": 0.7680605528836582 }, "h100": { "eta_bw": 0.8219516543325195, "eta_c": 0.43142716896141786, "mape": 0.06512007416540311, "n": 64, "attn_bw_frac": 0.7094333446982612 } } }, "forms": { "max": { "mape_logo": 0.1050090307005881, "mape_lomo": 0.07892129707615032, "mape_in": 0.062107935683446164 }, "sum": { "mape_logo": 0.10471184600439247, "mape_lomo": 0.07636385614385872, "mape_in": 0.062130243653889514 }, "smooth": { "mape_logo": 0.09579984855265163, "mape_lomo": 0.07895041723090473, "mape_in": 0.062217436246071 } }, "chosen_form": "smooth", "system": { "vLLM": { "stack": { "name": "vLLM", "eta_bw": 0.7744444064711272, "eta_c": 0.6078207723971903, "t0": 0.00224528080712071, "a": 8.05735049544692e-05, "combine": "smooth", "k": 4.0, "compute_mult": 1.0 }, "n": 188, "mape_in": 0.06090956456365651, "median_ape_in": 0.03841512643929196, "mape_logo": 0.10408696257168513, "per_logo": { "Nvidia A100 GPU": 0.17154376693971685, "Nvidia GH200 GPU": 0.05919784316579272, "Nvidia H100 GPU": 0.09292727708869816 }, "mape_lomo": 0.07562555735366443, "per_lomo": { "llama2-7b": 0.0837342216690668, "llama31-8b": 0.05565613538345523, "mistral-7b": 0.10507827738524675, "qwen2-7b": 0.06622822233467722 }, "boot": [ [ 0.8353708264730914, 0.47105157215421306, 0.0027255484484701832, 6.170605122120394e-05 ], [ 0.8560417085047962, 0.47256250640760716, 0.002961409476108579, 6.0721319163930246e-05 ], [ 0.7192319561221389, 0.9999999999999767, 0.001763079940410119, 0.0001033392349283213 ], [ 0.6732147229666561, 0.7499969451943471, 0.0015078635543028958, 8.77581078651131e-05 ], [ 0.7187804293985574, 0.6709995183143378, 0.00193480600409867, 7.802649296399806e-05 ], [ 0.6336665233933497, 0.9999999999999203, 0.001000113501736386, 0.00010146864654072308 ], [ 0.762585915760647, 0.7304400278917808, 0.0020895845707670675, 8.277739669134237e-05 ], [ 0.7637152296831576, 0.5510780691373647, 0.0021197671051368526, 8.221404141596497e-05 ], [ 0.8057501240953764, 0.43555030428069275, 0.0024921521654544887, 5.704186190180971e-05 ], [ 0.7953730920952746, 0.7505219748400214, 0.0022670700475717924, 8.176009474276736e-05 ], [ 0.762408366890393, 1.0, 0.0018911328362419529, 0.00012324202310205502 ], [ 0.8394238578831958, 0.6337529071861983, 0.0027684042093055923, 7.21363356895421e-05 ], [ 0.7845454745649316, 0.9999999999999997, 0.002226954989729916, 9.343958838078387e-05 ], [ 0.7460343115476997, 0.9999999999999916, 0.0019186310567728057, 0.00010155208638303724 ], [ 0.8124863414651788, 0.618531884685702, 0.002482982728969926, 8.881419000108848e-05 ], [ 0.7240993082499051, 0.45273853142594567, 0.0017701806621178843, 6.595019320135386e-05 ], [ 0.7437229377466746, 0.39850170399872864, 0.0022475527577927377, 5.3515794506564e-05 ], [ 0.7752307832959314, 0.9999999999999993, 0.002245687453103287, 8.886954308417157e-05 ], [ 0.7795571635310605, 0.5286024949660089, 0.0022372891319340053, 7.872142605615057e-05 ], [ 0.7680527664305951, 0.6099180212644428, 0.0022950893175969008, 7.572534932209138e-05 ], [ 0.761410225168744, 0.6272120926483548, 0.001993807502278899, 7.762453840081338e-05 ], [ 0.8622161899149313, 0.49811478146613464, 0.0030328311383799058, 6.796722329839808e-05 ], [ 0.7446388399982375, 0.654725305909102, 0.0018499409577999804, 8.797375812580262e-05 ], [ 0.8322357888797082, 0.4231799241132008, 0.002748950427211342, 6.474846511780691e-05 ], [ 0.8182308050526271, 0.6246315987283173, 0.002486184161428468, 9.229583610962463e-05 ], [ 0.8118260722446263, 0.45159914137064233, 0.0025363271072304113, 6.039399649669963e-05 ], [ 0.8685223966631739, 0.47116001273017516, 0.0030926640072680145, 6.0699402388866954e-05 ], [ 0.6571595307877908, 0.9999999999999994, 0.0011582699142094382, 8.513281425522218e-05 ], [ 0.7278670230804808, 0.9999999999995702, 0.0017357916610749636, 9.582268678246998e-05 ], [ 0.7368254823291168, 0.7129205115395021, 0.001798680710615292, 9.58517902652646e-05 ], [ 0.8486488254152718, 0.4446517267140769, 0.0029366332198873194, 6.828702180806769e-05 ], [ 0.804593219322101, 0.34121645646403054, 0.0023635973057661635, 3.793596918700034e-05 ], [ 0.8649213488536851, 0.35848340339718276, 0.0029338133914583652, 5.380104179468968e-05 ], [ 0.8287529615783064, 0.42574119654010273, 0.002701130706631773, 5.885317282745769e-05 ], [ 0.7874533953898625, 0.9999999999999949, 0.0023692546945181443, 9.065213289650417e-05 ], [ 0.8145593396763582, 0.33520137779190157, 0.0024431075639002233, 3.7507030225938545e-05 ], [ 0.7431869139144773, 0.9999999999999996, 0.0018308236077473816, 0.0001035999954949547 ], [ 0.8373484042291884, 0.41537752970705893, 0.0026703153410025026, 5.299019278415294e-05 ], [ 0.7036476176192261, 0.9999999999626071, 0.0016260211024298845, 0.00010666818772398499 ], [ 0.6128299265298791, 0.9999999999993938, 0.0009190937009442153, 0.000100747340240261 ], [ 0.7627861163692861, 1.0, 0.0020470553495592144, 9.984065357712759e-05 ], [ 0.7179290564225299, 0.9999999999999881, 0.0016265030761617122, 0.00010890743390330733 ], [ 0.751097475373388, 1.0, 0.001961121436625514, 0.00011847043552063087 ], [ 0.745551248332292, 1.0, 0.0019077165496051928, 0.00010219799443282238 ], [ 0.8165922055364906, 0.4392150842204382, 0.002633622389216885, 5.623422822369551e-05 ], [ 0.7157958911969412, 0.9999999999999709, 0.001447086476485912, 0.00012964889929157686 ], [ 0.8252952156096992, 0.8037686958692518, 0.002653637437900192, 7.3372367875923e-05 ], [ 0.7483286284513943, 0.9999999999995427, 0.001931809832949121, 0.00010594651432441188 ], [ 0.7616234179358646, 0.4345319046492305, 0.0019220165327800236, 6.233636153154775e-05 ], [ 0.7668977975115893, 0.9999999999999997, 0.002030100975435476, 9.803899154798448e-05 ], [ 0.7313087138205849, 0.6262403053719064, 0.0019925301126118666, 7.695607319257882e-05 ], [ 0.8335318018607543, 0.8431573698607245, 0.002602883146166928, 9.606342885439337e-05 ], [ 0.702750393739161, 0.7193248097810666, 0.0017167931539233707, 8.193025270493273e-05 ], [ 0.8114576744123023, 0.5794606766352621, 0.0024743261548782524, 8.333277949324936e-05 ], [ 0.8336212760159437, 0.43946072679429166, 0.0026990316077750197, 6.211519357393451e-05 ], [ 0.8077019755454314, 0.44596306849928063, 0.002589685958769796, 5.7638899585390255e-05 ], [ 0.8093518137624087, 0.4369821925777495, 0.0026809553564767336, 6.283914858591618e-05 ], [ 0.7895076969641859, 0.9999999999995145, 0.0024872267137695733, 9.775805700950685e-05 ], [ 0.8289441481528866, 0.4885924738367364, 0.002909517178506294, 6.462126736195849e-05 ], [ 0.6806983667885629, 0.2354614568986886, 0.0018175080914621384, 3.108332427879116e-05 ], [ 0.7901713822281731, 0.9999999999999996, 0.0022842597039831555, 9.627723001798349e-05 ], [ 0.8292701345021662, 0.4217254769703908, 0.002682192884883911, 5.84177041201604e-05 ], [ 0.8050992509995542, 0.9999999999966119, 0.0025440847196868148, 9.033324253165938e-05 ], [ 0.7748350905770086, 0.6761130904987867, 0.0021473466731111307, 8.56910469834146e-05 ], [ 0.8610689971531106, 0.5383361980554172, 0.0029611932157315992, 7.476419909586163e-05 ], [ 0.805034205571647, 0.5713597259576557, 0.002507380147929796, 8.425292972567204e-05 ], [ 0.8118523090850409, 0.9999999999296268, 0.0024975023253337577, 9.164177486735534e-05 ], [ 0.7385630253698925, 0.9999999999995822, 0.0018563904926071125, 7.483260928731443e-05 ], [ 0.7357301703687016, 0.6290900467531724, 0.0018497842142061395, 7.947504722989657e-05 ], [ 0.7400445485327132, 0.765249832904814, 0.0017236662747003352, 8.305813890766016e-05 ], [ 0.7447748013099332, 0.999999999993609, 0.0019473445298185319, 9.948531308470131e-05 ], [ 0.8174676987931586, 0.4220654478735278, 0.002463385948851147, 5.1052571288925954e-05 ], [ 0.7791356346799273, 0.5859463712452607, 0.002244081315439596, 9.213863530673593e-05 ], [ 0.7605841645461007, 0.7445174548852463, 0.0020964306623522406, 9.126751756419745e-05 ], [ 0.7946798515668119, 0.45851446596260326, 0.0023996294041861437, 7.315791955377272e-05 ], [ 0.8111236612532428, 0.4118507483848411, 0.00245276392945004, 4.9900623529653874e-05 ], [ 0.7611940655504937, 0.5499033917533854, 0.002188298304902807, 8.755407450365254e-05 ], [ 0.7186789156176874, 0.4831799218029921, 0.001675574859959616, 7.556260621040713e-05 ], [ 0.7535823358380767, 0.6431608500416047, 0.001727367850641587, 7.881795271914563e-05 ], [ 0.6964303540335867, 0.9999999999999988, 0.0014114773169190916, 9.554030654954195e-05 ], [ 0.8622954195157714, 0.42808983040346976, 0.0029568213774586747, 6.142656647115003e-05 ], [ 0.6532902540678565, 0.9091396803803754, 0.0011555060786130388, 7.976621590795323e-05 ], [ 0.7737072384941597, 0.788970032594236, 0.0021948117595252357, 8.274181972773067e-05 ], [ 0.6913857253090951, 0.9999999999999989, 0.0017955563282699466, 0.00010050174936765267 ], [ 0.7361599624177599, 0.6971480138244244, 0.0018686850365184032, 9.33442892049338e-05 ], [ 0.7651850789572118, 0.4968597510282423, 0.002218988525801107, 7.187077115291988e-05 ], [ 0.8317015590712944, 0.335980105480587, 0.002794223650601909, 5.1635127271601796e-05 ], [ 0.705898435285506, 0.6091458624739697, 0.0016656074809989383, 7.52033184046103e-05 ], [ 0.7242327325689151, 0.9999999999999823, 0.0018379501960491915, 0.00010109350235933514 ], [ 0.7706957175900865, 0.6443317128363204, 0.0022246193673120434, 7.468948147003183e-05 ], [ 0.8512860626912084, 0.4218019565341914, 0.0028977410778572585, 5.953210212922031e-05 ], [ 0.706332070657838, 0.9999999944515124, 0.0015131808512747439, 9.895078837013193e-05 ], [ 0.7301575713074908, 0.8618359808744479, 0.0019391214088100457, 8.172832042258141e-05 ], [ 0.8141730966056073, 0.4737851176021522, 0.0024554167927473987, 6.0073564933500756e-05 ], [ 0.7842760970160346, 0.3393907460739776, 0.002358984204081886, 4.4110643310761415e-05 ], [ 0.723993961770651, 0.9909100299481836, 0.00199901670544235, 9.219534857777653e-05 ], [ 0.7514454909803882, 0.4691294332652259, 0.002307517481031161, 7.66758301008121e-05 ], [ 0.8275399512959896, 0.9999999999999958, 0.0026332768419447907, 9.825805788544647e-05 ], [ 0.7680779687604855, 0.5330922293290868, 0.0020988254960352163, 8.129740948142792e-05 ], [ 0.8420273314466622, 0.31712452880032577, 0.002859914285645426, 4.220790785411018e-05 ], [ 0.7417991233248284, 0.9647496573111105, 0.0017019898693873498, 0.00010955535435716848 ], [ 0.756580377041966, 0.7706830282418149, 0.0022377121362757964, 8.354763202470197e-05 ], [ 0.7449406057444884, 0.9999999999820625, 0.0021922727762014316, 9.118138355914767e-05 ], [ 0.7029170537006624, 0.9999999999999976, 0.0015753956910101261, 8.754437151289621e-05 ], [ 0.7296784074867149, 0.9999999999999999, 0.0020297735984073687, 0.00010100941472724924 ], [ 0.7407855359134962, 0.9999999999985886, 0.0018694943278335292, 0.0001064929144036425 ], [ 0.7133422731531179, 0.9999999999997181, 0.0016472569475700259, 9.235927281590073e-05 ], [ 0.8253355966144139, 0.4041447387681608, 0.002703288621616843, 5.516155032660833e-05 ], [ 0.8167212633173931, 0.4766319842712363, 0.002618706568117999, 6.304278328280155e-05 ], [ 0.7158788070740595, 0.9999999999999968, 0.0013906219408478344, 0.00012274607843641716 ], [ 0.8041450533891074, 0.4092959536674991, 0.0024846603250295463, 5.259734973337097e-05 ], [ 0.7412798760308232, 0.9999999999988626, 0.0017782044825136064, 0.00011309770086046463 ], [ 0.7137527705871421, 0.9991264209171432, 0.0015776641633933914, 7.962165825284975e-05 ], [ 0.7818082905647621, 0.5713710361524574, 0.0022885028130101574, 7.39540333522652e-05 ], [ 0.8477608436115328, 0.42149751095011684, 0.0028425123179948064, 6.059951463762912e-05 ], [ 0.6281083087184094, 0.9999999999999984, 0.0007587490349252905, 0.00010949461564514614 ], [ 0.8677153547247751, 0.4014058929512122, 0.002893777312471754, 5.047659227148908e-05 ], [ 0.8162801613692985, 0.41903257575738434, 0.002602130316998014, 5.308643895688496e-05 ], [ 0.8319358917083393, 0.46940619107168374, 0.0028687296164740094, 6.859572607730729e-05 ], [ 0.7669409677834198, 0.5417765464816305, 0.002084972134328352, 7.670243641967474e-05 ] ] }, "TensorRT-LLM": { "stack": { "name": "TensorRT-LLM", "eta_bw": 0.7796191116388631, "eta_c": 0.5606647598235109, "t0": 0.0013746118851451557, "a": 1.8938656021816108e-05, "combine": "smooth", "k": 4.0, "compute_mult": 1.0 }, "n": 184, "mape_in": 0.06355373992158145, "median_ape_in": 0.053024128247834035, "mape_logo": 0.08733257988103045, "per_logo": { "Nvidia A100 GPU": 0.06048933463173345, "Nvidia GH200 GPU": 0.10745149858776595, "Nvidia H100 GPU": 0.09019329973399003 }, "mape_lomo": 0.08234755667069374, "per_lomo": { "llama2-7b": 0.13072963367151963, "llama31-8b": 0.05876058975972311, "mistral-7b": 0.057166160953662414, "qwen2-7b": 0.11098690323387012 }, "boot": [ [ 0.7359219060193969, 0.6227365856397531, 0.001053278155594566, 1.834942753793523e-05 ], [ 0.7885445651941824, 0.7002023798439808, 0.001515009491165306, 2.9313571695961898e-05 ], [ 0.7432319243228959, 0.5399354597491807, 0.0011267602156766936, 1.5776290419157943e-05 ], [ 0.7499636116041716, 0.5298761221743113, 0.0011553055544849522, 1.6507997215328654e-05 ], [ 0.7858095594054194, 0.49034586631762084, 0.001415765333832417, 1.9052781687999997e-05 ], [ 0.6659882200791357, 0.9999999999999986, 0.0005264585746758356, 3.3728417654820376e-05 ], [ 0.8232976297749337, 0.6712797966472696, 0.0016645310458951142, 2.6053189738141698e-05 ], [ 0.758756186213661, 0.7030367513781075, 0.0013547209015168553, 2.931548102677105e-05 ], [ 0.8139827763083345, 0.46182580864553724, 0.001561587478925132, 1.0802003097855554e-05 ], [ 0.8526462814080589, 0.5046045557860185, 0.0018467323350649077, 1.65781482233843e-05 ], [ 0.7903879741691824, 0.9965980397636426, 0.0014619180521554238, 3.2973836048759415e-05 ], [ 0.8470741951105514, 0.531680564869698, 0.0016361563702348147, 1.654043762483558e-05 ], [ 0.8026086634168096, 0.8200176822348986, 0.0015270956422075031, 2.781636558752014e-05 ], [ 0.751799405712363, 1.0, 0.0012610272690656032, 4.051760859376816e-05 ], [ 0.8115508580888094, 0.48813434111464693, 0.0015579185860161429, 6.978328689059467e-06 ], [ 0.760334508444434, 0.5826573516308334, 0.0013268766161577646, 2.320932391615385e-05 ], [ 0.8435593242371586, 0.44451850836012785, 0.0017184877419096965, 1.5701646099380192e-05 ], [ 0.7765546474593993, 0.8794054255782748, 0.0013174421211071471, 3.309285400480369e-05 ], [ 0.768080705357406, 0.5758941419278004, 0.0013104836156245735, 1.9745000172613383e-05 ], [ 0.7630169244605285, 0.5068682976507168, 0.001340881463799197, 1.6837494080019053e-05 ], [ 0.7234398912724461, 0.573109346315636, 0.0010340838026915058, 1.5308655680066116e-05 ], [ 0.7971500897463571, 0.4894992663635182, 0.0014570099802011557, 9.640009955068527e-06 ], [ 0.834911800694022, 0.4934402044958195, 0.0017472790278948154, 1.7404888739696767e-05 ], [ 0.7827751995650781, 0.4659411928817351, 0.001330165036281363, 6.457778824963409e-07 ], [ 0.7915972128682512, 0.4775241517962862, 0.001448144443124696, 3.565784779119575e-06 ], [ 0.8128980212312822, 0.4135290833262471, 0.001510673190796661, 8.395170816124575e-07 ], [ 0.8091767385355841, 0.6304231157866583, 0.001685425855539217, 2.7179788072376374e-05 ], [ 0.783937619703882, 1.0, 0.0015171424426579912, 3.774421916117577e-05 ], [ 0.7613046815898579, 0.6576995850345853, 0.0012381535663196018, 2.282769243470272e-05 ], [ 0.7484657588115747, 0.6717938586675668, 0.0011313392822102808, 2.2891359986181538e-05 ], [ 0.7959984931409848, 0.4684957150209964, 0.0013900071165396403, 3.2879226228400483e-06 ], [ 0.8304343779840485, 0.470944294346478, 0.001836150755481388, 1.3122064174162856e-05 ], [ 0.7901993636286831, 0.413821033441648, 0.0014286473029545593, 1.0000000000000952e-07 ], [ 0.8100899255017717, 0.46216850403620807, 0.0015168179900140204, 1.0342775595904172e-05 ], [ 0.7365152872127152, 0.9999999999999887, 0.0010145055881416946, 3.292945153348023e-05 ], [ 0.8520093980350707, 0.4274670043681462, 0.001984447832755207, 5.617686048134292e-06 ], [ 0.7428640266570005, 0.4825978275469581, 0.001077680385525213, 8.834720183401542e-06 ], [ 0.7190263269667936, 0.5728198592661821, 0.000979490014570719, 1.4873861186753387e-05 ], [ 0.7112398326070942, 0.9999999999983566, 0.0008451237684925684, 3.4670860887818015e-05 ], [ 0.7065429802012511, 0.6325046718388212, 0.0008242410312853946, 1.719582782345495e-05 ], [ 0.8356567619051908, 0.9999999999924324, 0.0016384599095415357, 3.122209025222083e-05 ], [ 0.6979870098214077, 0.732133670432916, 0.0007525411928219548, 2.247419244634168e-05 ], [ 0.7065010164918067, 0.6172526914523867, 0.0007428289480710296, 1.6555994989421926e-05 ], [ 0.8056332470385152, 0.9999999999999751, 0.0016329476228324155, 3.977178354824445e-05 ], [ 0.7201199757358795, 0.99999999999971, 0.0009850838375090535, 3.777728157439849e-05 ], [ 0.69927732241733, 0.9999999999999643, 0.0007116986184844294, 3.4860400778507605e-05 ], [ 0.7803790957127047, 0.9999999999999574, 0.0014027531392505224, 3.326422778272373e-05 ], [ 0.7661527003246358, 0.7943431057516055, 0.001254319812643982, 3.1811972812687906e-05 ], [ 0.7760257670909669, 0.5750869364747103, 0.001404632921053081, 1.941069488591985e-05 ], [ 0.8898660567295134, 0.9999999999985952, 0.0022114878608227007, 3.855160120541064e-05 ], [ 0.7648704776473704, 0.5080390415265577, 0.001249024257025083, 1.820387944466332e-05 ], [ 0.8417112473731513, 0.5096572619601326, 0.0016782393920205892, 7.328228004955635e-06 ], [ 0.6895866145742238, 0.9999999999999999, 0.0007239117976723822, 3.404160436925011e-05 ], [ 0.8254367068419877, 0.5058148114887024, 0.0017098496589789986, 1.331734037226613e-05 ], [ 0.7848321372702044, 0.4356672433878565, 0.0012601637414879879, 1.0000000000160221e-07 ], [ 0.7269737173893612, 0.999999999998833, 0.0010293547324274762, 3.5910463604801426e-05 ], [ 0.7265894904279827, 0.5853620371839363, 0.0009325137882295526, 1.753779234576245e-05 ], [ 0.8254779861590845, 0.5181472157217868, 0.0014934517645343824, 1.2850286309177061e-05 ], [ 0.7537925020649257, 0.5314537955317353, 0.0011756104198489172, 1.980757319069345e-05 ], [ 0.8144320762928754, 0.45714579161213753, 0.0015053091358659852, 1.5810403112121225e-05 ], [ 0.7956377300741353, 1.0, 0.0014454961889524116, 3.5998686861516885e-05 ], [ 0.7711906525563296, 0.5312913210334158, 0.0012691796101071344, 1.3398300250400527e-05 ], [ 0.8399496518558034, 0.9999999999999056, 0.0016931163661550843, 3.286299120293595e-05 ], [ 0.7768209723035081, 0.9999999999999976, 0.0013966762342397117, 4.0216862140214555e-05 ], [ 0.8221925579807335, 0.437592342708557, 0.0014391233000413252, 1.0000000000000029e-07 ], [ 0.784011116539838, 0.4800298966977662, 0.0013010038449956973, 8.441174462496427e-06 ], [ 0.8775779936103942, 0.5083750624655717, 0.0017191795157205578, 1.2527105672826686e-05 ], [ 0.7997124543595263, 0.30233032110386293, 0.001541994971532104, 1.0000000000000029e-07 ], [ 0.7577805373665452, 0.720488710529132, 0.0012706445077702421, 3.151896458602727e-05 ], [ 0.7619044346287372, 0.9999999999981702, 0.0014283086987594237, 4.5271636415576855e-05 ], [ 0.7687546437843186, 0.6377422261231228, 0.0013249416467350087, 1.986404515905207e-05 ], [ 0.7941456263215518, 0.4574385640028715, 0.0014213239595069402, 1.0000000000002125e-07 ], [ 0.7598341949845947, 0.5079036734504417, 0.0011821323710949577, 7.648275036592302e-06 ], [ 0.7429308064473075, 0.9444429665529903, 0.001102928945144453, 3.5360096354974814e-05 ], [ 0.8190997348548839, 0.3884618741932314, 0.0015654652380597742, 1.0719586776082247e-06 ], [ 0.8059943331594777, 0.4424694045761098, 0.0015840076508554556, 6.050835303748424e-06 ], [ 0.719865245704601, 0.6636696789044534, 0.0009477702814159146, 2.121595255939478e-05 ], [ 0.7793834926070883, 0.4234041082720739, 0.0013640231636910761, 7.383401773389143e-06 ], [ 0.7525597368912674, 0.8543646551563538, 0.0012632938214623018, 3.468872916761552e-05 ], [ 0.7560839140913481, 0.8369319979774522, 0.0013494177505132812, 3.4931437317347206e-05 ], [ 0.8437059232715044, 0.5266934669738106, 0.0016777630829972164, 1.590160662198321e-05 ], [ 0.7634444706048117, 0.31492066861343165, 0.0012763461356467116, 1.0000000000015306e-07 ], [ 0.8380862083620546, 0.4472476417910155, 0.0016637926189417688, 1.2063864643635252e-05 ], [ 0.7363367602687672, 0.8289348752461639, 0.000986520662114427, 2.481618885728654e-05 ], [ 0.7520776999078977, 0.4837314440589353, 0.0011867935585451093, 9.353217317409197e-06 ], [ 0.8000823205279449, 0.45050350388382826, 0.001573639646426504, 1.2098739118247222e-05 ], [ 0.8252042734719364, 0.3962107744813243, 0.001602737738552992, 1.3722288375878344e-06 ], [ 0.7419650665394701, 0.706327245460553, 0.0012623146517083666, 2.9666503132031772e-05 ], [ 0.7896441665999822, 0.5584682213433155, 0.0014127322852043338, 2.2833947822427677e-05 ], [ 0.6674822656775086, 0.8793291412156449, 0.0005370436011334892, 2.8288532645785582e-05 ], [ 0.7346417727676701, 0.6337822945032506, 0.001020886307425816, 1.8396637439716057e-05 ], [ 0.7360743377164395, 0.5841923766414739, 0.0011089470944841836, 2.0417576782966276e-05 ], [ 0.7126160023652109, 0.6886051823551841, 0.0009077134322523333, 2.700976313924168e-05 ], [ 0.7714723359763004, 0.4937640147969343, 0.0011586129298749549, 3.883175743291744e-06 ], [ 0.8254486866465843, 0.39384736620931093, 0.0015919442657151939, 6.730213342437136e-06 ], [ 0.8253500061479037, 0.48485482846841094, 0.0015151849760112852, 1.1520243303471474e-05 ], [ 0.8415705456830406, 0.37811287884328404, 0.0015250092366529055, 5.119151740533511e-07 ], [ 0.8526500988934947, 0.9816467669098754, 0.0017217163936249266, 2.7795884187620053e-05 ], [ 0.7419023457574205, 0.7321025197721713, 0.0011681336520221288, 2.8179204994797988e-05 ], [ 0.7783633958463528, 0.5428464344308617, 0.0015035444762186783, 2.129950596243695e-05 ], [ 0.7447248999192965, 0.8778259706362875, 0.0011536950388628482, 3.696605766256025e-05 ], [ 0.7580758537312632, 0.4962393182420136, 0.0011882732389728377, 1.682546465730765e-05 ], [ 0.737455449051877, 0.7977881012995194, 0.0010779312834962255, 2.9420142280448206e-05 ], [ 0.9100792054522403, 0.8533589210609352, 0.002338550858508574, 3.693662305639224e-05 ], [ 0.74658189201559, 0.505100402976335, 0.0011131708095762519, 1.3066249722336136e-05 ], [ 0.7914905790736605, 0.48073497903346135, 0.0013836792155246785, 9.366211053315413e-06 ], [ 0.7989305870757035, 0.624486634705717, 0.0015808318922203085, 2.783806930671276e-05 ], [ 0.794128635252851, 0.4455928793903156, 0.0015450773629228508, 9.219772601006449e-06 ], [ 0.7465498943576939, 0.4764094050322832, 0.0011061823094758508, 8.63370136640782e-06 ], [ 0.7142669948736567, 0.6448001191183055, 0.0008943179654972464, 2.0758293853005927e-05 ], [ 0.8132616770214822, 0.41317089056816414, 0.001649124644236535, 5.888296839981448e-06 ], [ 0.7407232734625357, 0.6693768560532652, 0.0010886794117472467, 2.5947620197189985e-05 ], [ 0.7329436977209065, 0.9999999999999998, 0.0012336768050582373, 4.416409348610507e-05 ], [ 0.7834387879262319, 0.4615690136590909, 0.0014935015280803838, 1.4460166143241233e-05 ], [ 0.8422398528440034, 0.48881906719890417, 0.0016974229104824645, 9.307665500048196e-06 ], [ 0.7107783003182624, 1.0, 0.0009482983387662579, 3.800856299521965e-05 ], [ 0.8381838756926461, 0.7572702052173461, 0.0018686303657684154, 2.967223256463352e-05 ], [ 0.721901273874388, 1.0, 0.0010124715863267235, 3.7807850117186755e-05 ], [ 0.7885907248061383, 0.5431761652438593, 0.0013449329925741143, 1.5427865437288046e-05 ], [ 0.754995569387005, 0.47007721296885274, 0.0012243525776440935, 6.314859963572301e-06 ] ] } }, "system_max": { "vLLM": { "stack": { "name": "vLLM", "eta_bw": 0.7751549369880121, "eta_c": 0.5963988791049138, "t0": 0.002249833167186188, "a": 8.025375061343338e-05, "combine": "max", "k": 4.0, "compute_mult": 1.0 }, "n": 188, "mape_in": 0.06084518653047605, "median_ape_in": 0.03780072399981438, "mape_logo": 0.11926649440533145, "per_logo": { "Nvidia A100 GPU": 0.2266343546900922, "Nvidia GH200 GPU": 0.059156419509656936, "Nvidia H100 GPU": 0.09281669809412675 }, "mape_lomo": 0.07559524226767847, "per_lomo": { "llama2-7b": 0.08376126193382903, "llama31-8b": 0.055711396795361266, "mistral-7b": 0.10481594492100356, "qwen2-7b": 0.06622620466966624 } }, "TensorRT-LLM": { "stack": { "name": "TensorRT-LLM", "eta_bw": 0.7836849553838169, "eta_c": 0.527010598403318, "t0": 0.0014030260698517777, "a": 1.7489411415306152e-05, "combine": "max", "k": 4.0, "compute_mult": 1.0 }, "n": 184, "mape_in": 0.06339813590495912, "median_ape_in": 0.052708378327944005, "mape_logo": 0.09044162213269816, "per_logo": { "Nvidia A100 GPU": 0.07152040411015496, "Nvidia GH200 GPU": 0.10737842605540904, "Nvidia H100 GPU": 0.09023110029341469 }, "mape_lomo": 0.08231965742393678, "per_lomo": { "llama2-7b": 0.13074280667924645, "llama31-8b": 0.05834596296526117, "mistral-7b": 0.0575047794778978, "qwen2-7b": 0.1109051767434791 } } }, "v3": { "eta": 0.7859524301863509, "t0": 0.0043992205190368965, "n_train": 10, "n_test": 18, "spearman": 0.9896800825593396, "median_ape": 0.19641776665455046, "mape": 0.19738789249080585, "table": [ { "gpu": "3070 8GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 448, "value": 70.94, "pred": 57.30337895944639, "eta_obs": 1.057837468781738 }, { "gpu": "3080 10GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 760, "value": 106.4, "pred": 82.69323238120502, "eta_obs": 1.2095465451063712 }, { "gpu": "3080 Ti 12GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 912, "value": 106.71, "pred": 92.50171726743433, "eta_obs": 1.0134905810408072 }, { "gpu": "4070 Ti 12GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 504, "value": 82.21, "pred": 62.49694296870206, "eta_obs": 1.1743167745803165 }, { "gpu": "4080 16GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 717, "value": 106.22, "pred": 79.65402450867047, "eta_obs": 1.2780141109037009 }, { "gpu": "4080 16GB", "cfg": "8B F16", "tier": "consumer", "bw": 717, "value": 40.29, "pred": 32.20750358208973, "eta_obs": 1.0256777871584486 }, { "gpu": "RTX 4000 Ada 20GB", "cfg": "8B Q4_K_M", "tier": "workstation", "bw": 360, "value": 58.59, "pred": 48.44630874510202, "eta_obs": 1.0076605390698568 }, { "gpu": "RTX 4000 Ada 20GB", "cfg": "8B F16", "tier": "workstation", "bw": 360, "value": 20.85, "pred": 17.398556839598413, "eta_obs": 0.9576111230853452 }, { "gpu": "3090 24GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 936, "value": 111.74, "pred": 93.93009181577683, "eta_obs": 1.0790559863278681 }, { "gpu": "3090 24GB", "cfg": "8B F16", "tier": "consumer", "bw": 936, "value": 46.51, "pred": 40.300840020216285, "eta_obs": 0.9381941408146872 }, { "gpu": "4090 24GB", "cfg": "8B Q4_K_M", "tier": "consumer", "bw": 1008, "value": 127.74, "pred": 98.03920051264187, "eta_obs": 1.3295122291030945 }, { "gpu": "4090 24GB", "cfg": "8B F16", "tier": "consumer", "bw": 1008, "value": 54.34, "pred": 42.816972424501, "eta_obs": 1.0639191901645226 }, { "gpu": "RTX 5000 Ada 32GB", "cfg": "8B Q4_K_M", "tier": "workstation", "bw": 576, "value": 89.87, "pred": 68.72574792609842, "eta_obs": 1.1858702586005574 }, { "gpu": "RTX 5000 Ada 32GB", "cfg": "8B F16", "tier": "workstation", "bw": 576, "value": 32.67, "pred": 26.615403733029922, "eta_obs": 0.9947540562687923 }, { "gpu": "RTX A6000 48GB", "cfg": "8B Q4_K_M", "tier": "workstation", "bw": 768, "value": 102.22, "pred": 83.24491571868943, "eta_obs": 1.1114993026267512 }, { "gpu": "RTX A6000 48GB", "cfg": "8B F16", "tier": "workstation", "bw": 768, "value": 40.25, "pred": 34.154200347894026, "eta_obs": 0.9564111428518414 }, { "gpu": "RTX 6000 Ada 48GB", "cfg": "8B Q4_K_M", "tier": "workstation", "bw": 960, "value": 130.99, "pred": 95.32851540948249, "eta_obs": 1.479804889536415 }, { "gpu": "RTX 6000 Ada 48GB", "cfg": "8B F16", "tier": "workstation", "bw": 960, "value": 51.97, "pred": 41.14714179812888, "eta_obs": 1.0539521864715458 }, { "gpu": "A40 48GB", "cfg": "8B Q4_K_M", "tier": "datacenter", "bw": 696, "value": 88.95, "pred": 78.12285339856344, "eta_obs": 0.9649044022737184 }, { "gpu": "A40 48GB", "cfg": "8B F16", "tier": "datacenter", "bw": 696, "value": 33.95, "pred": 31.39447008008646, "eta_obs": 0.8611623236650897 }, { "gpu": "L40S 48GB", "cfg": "8B Q4_K_M", "tier": "datacenter", "bw": 864, "value": 113.6, "pred": 89.55118219341863, "eta_obs": 1.2078750724185845 }, { "gpu": "L40S 48GB", "cfg": "8B F16", "tier": "datacenter", "bw": 864, "value": 43.42, "pred": 37.715129354229504, "eta_obs": 0.9329078264936257 }, { "gpu": "A100 PCIe 80GB", "cfg": "8B Q4_K_M", "tier": "datacenter", "bw": 1935, "value": 138.31, "pred": 134.7523157254057, "eta_obs": 0.8389489151337184 }, { "gpu": "A100 PCIe 80GB", "cfg": "8B F16", "tier": "datacenter", "bw": 1935, "value": 54.56, "pred": 70.05756470343105, "eta_obs": 0.5571801713133513 }, { "gpu": "A100 SXM 80GB", "cfg": "8B Q4_K_M", "tier": "datacenter", "bw": 2039, "value": 133.38, "pred": 137.6103638065023, "eta_obs": 0.7274829914101165 }, { "gpu": "A100 SXM 80GB", "cfg": "8B F16", "tier": "datacenter", "bw": 2039, "value": 53.18, "pred": 72.62000513081995, "eta_obs": 0.511302461145103 }, { "gpu": "H100 PCIe 80GB", "cfg": "8B Q4_K_M", "tier": "datacenter", "bw": 2039, "value": 144.49, "pred": 137.6103638065023, "eta_obs": 0.8937934329551209 }, { "gpu": "H100 PCIe 80GB", "cfg": "8B F16", "tier": "datacenter", "bw": 2039, "value": 67.79, "pred": 72.62000513081995, "eta_obs": 0.711464047338404 } ] }, "argonne_rows": { "total": 400, "infeasible": 4, "pressured": 24 } }
{
"vidur": {
"max": {
"a100": {
"eta_bw": 0.7310774976236001,
"eta_c": 0.6654064678288377,
"mape": 0.057558084397660726,
"n": 140,
"attn_bw_frac": 0.5991907381812237
},
"a40": {
"eta_bw": 0.7536243228217148,
"eta_c": 0.6,
"mape": 0.05581423683930286,
"n": 64,
"attn_bw_frac": 0.7680605528836582
},
"h100": {
"eta_bw": 0.8114088587438508,
"eta_c": 0.6,
"mape": 0.064630345092196,
"n": 64,
"attn_bw_frac": 0.7094333446982612
}
},
"sum": {
"a100": {
"eta_bw": 0.8154750554470435,
"eta_c": 1.2,
"mape": 0.06514696945554,
"n": 140,
"attn_bw_frac": 0.5991907381812237
},
"a40": {
"eta_bw": 0.8202894751055775,
"eta_c": 1.2,
"mape": 0.045614744672416406,
"n": 64,
"attn_bw_frac": 0.7680605528836582
},
"h100": {
"eta_bw": 0.8671763154735798,
"eta_c": 1.199999999999998,
"mape": 0.06913968586498931,
"n": 64,
"attn_bw_frac": 0.7094333446982612
}
},
"smooth": {
"a100": {
"eta_bw": 0.7500135866949069,
"eta_c": 0.6454558433375267,
"mape": 0.03124893839795496,
"n": 140,
"attn_bw_frac": 0.5991907381812237
},
"a40": {
"eta_bw": 0.7675289933621684,
"eta_c": 0.496990333497917,
"mape": 0.04068866140227775,
"n": 64,
"attn_bw_frac": 0.7680605528836582
},
"h100": {
"eta_bw": 0.8219516543325195,
"eta_c": 0.43142716896141786,
"mape": 0.06512007416540311,
"n": 64,
"attn_bw_frac": 0.7094333446982612
}
}
},
"forms": {
"max": {
"mape_logo": 0.1050090307005881,
"mape_lomo": 0.07892129707615032,
"mape_in": 0.062107935683446164
},
"sum": {
"mape_logo": 0.10471184600439247,
"mape_lomo": 0.07636385614385872,
"mape_in": 0.062130243653889514
},
"smooth": {
"mape_logo": 0.09579984855265163,
"mape_lomo": 0.07895041723090473,
"mape_in": 0.062217436246071
}
},
"chosen_form": "smooth",
"system": {
"vLLM": {
"stack": {
"name": "vLLM",
"eta_bw": 0.7744444064711272,
"eta_c": 0.6078207723971903,
"t0": 0.00224528080712071,
"a": 8.05735049544692e-05,
"combine": "smooth",
"k": 4.0,