Further improvments to support gguf

#13
Files changed (1) hide show
  1. index.html +386 -514
index.html CHANGED
@@ -1,287 +1,280 @@
1
  <!DOCTYPE html>
2
  <html lang="en">
3
- <head>
4
- <meta charset="UTF-8" />
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
- <script>
7
- function strToHtml(str) {
8
- let parser = new DOMParser();
9
- return parser.parseFromString(str, "text/html");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  }
11
 
12
- //Short, jQuery-independent function to read html table and write them into an Array.
13
- //Kudos to RobG at StackOverflow
14
- function tableToObj(table) {
15
- var rows = table.rows;
16
- var propCells = rows[0].cells;
17
- var propNames = [];
18
- var results = [];
19
- var obj, row, cells;
20
 
21
- // Use the first row for the property names
22
- // Could use a header section but result is the same if
23
- // there is only one header row
24
- for (var i = 0, iLen = propCells.length; i < iLen; i++) {
25
- propNames.push(
26
- (propCells[i].textContent || propCells[i].innerText).trim()
27
- );
28
  }
 
 
 
 
29
 
30
- // Use the rows for data
31
- // Could use tbody rows here to exclude header & footer
32
- // but starting from 1 gives required result
33
- for (var j = 1, jLen = rows.length; j < jLen; j++) {
34
- cells = rows[j].cells;
35
- obj = {};
36
 
37
- for (var k = 0; k < iLen; k++) {
38
- obj[propNames[k]] = (
39
- cells[k].textContent || cells[k].innerText
40
- ).trim();
41
- }
42
- results.push(obj);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  }
44
- return results;
45
  }
46
-
47
- function formatGpu(gpus) {
48
- return gpus.map(
49
- (g) => `${g["Product Name"]} - ${g["Memory"].split(",")[0]}`
50
- );
51
  }
52
-
53
- const gguf_quants = {
54
- "IQ1_S": 1.56,
55
- "IQ2_XXS": 2.06,
56
- "IQ2_XS": 2.31,
57
- "IQ2_S": 2.5,
58
- "IQ2_M": 2.7,
59
- "IQ3_XXS": 3.06,
60
- "IQ3_XS": 3.3,
61
- "Q2_K": 3.35,
62
- "Q3_K_S": 3.5,
63
- "IQ3_S": 3.5,
64
- "IQ3_M": 3.7,
65
- "Q3_K_M": 3.91,
66
- "Q3_K_L": 4.27,
67
- "IQ4_XS": 4.25,
68
- "IQ4_NL": 4.5,
69
- "Q4_0": 4.55,
70
- "Q4_K_S": 4.58,
71
- "Q4_K_M": 4.85,
72
- "Q5_0": 5.54,
73
- "Q5_K_S": 5.54,
74
- "Q5_K_M": 5.69,
75
- "Q6_K": 6.59,
76
- "Q8_0": 8.5,
77
  }
78
-
79
- async function modelConfig(hf_model, hf_token) {
80
- auth = hf_token == "" ? {} : {
81
- headers: {
82
- 'Authorization': `Bearer ${hf_token}`
83
- }
84
- }
85
- let config_res = await fetch(
86
- `https://huggingface.co/${hf_model}/raw/main/config.json`, auth
87
- )
88
- if (config_res.status === 401) {
89
- throw new Error("Model is either private or gated, you must provide an access token")
90
- }
91
- if (config_res.status === 403) {
92
- throw new Error("Model is either private or gated and provided access token does not have access to the repo")
93
- }
94
- let config = await config_res.json()
95
- let model_size = 0
96
- let dtype = parseInt(config["torch_dtype"].replaceAll(/\D/g, '')) / 8
97
- if ("text_config" in config) {
98
- config = config["text_config"]
99
  }
 
100
  try {
101
- model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/model.safetensors.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype
102
  if (isNaN(model_size)) {
103
- throw new Error("no size in safetensors metadata")
104
  }
105
- } catch (e) {
106
  try {
107
- model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/pytorch_model.bin.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype
 
 
108
  if (isNaN(model_size)) {
109
  throw new Error("no size in pytorch metadata")
110
  }
111
  } catch {
112
- try {
113
- model_size = (await fetch(
114
- `https://huggingface.co/api/models/${hf_model}`
115
- ).then(r => r.json()))["safetensors"]["total"]
116
- if (isNaN(model_size)) {
117
- throw new Error("no size in pytorch metadata")
118
- }
119
- } catch {
120
- throw new Error("Couldn't determine model size from safetensor/pytorch index metadata nor from the model card. If the model is an unsharded pytorch model, it is not supported by this calculator.")
121
- }
122
  }
123
  }
124
- config.parameters = model_size
125
- return config
126
  }
 
 
 
127
 
128
- function inputBuffer(context=8192, model_config, bsz=512) {
129
- /* Calculation taken from github:ggerganov/llama.cpp/llama.cpp:11248
130
- ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
131
- ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);
132
- ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
133
- ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);
134
- ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);
135
- ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch);
136
 
137
- n_embd is hidden size (github:ggeranov/llama.cpp/convert.py:248)
138
- */
139
- const inp_tokens = bsz
140
- const inp_embd = model_config["hidden_size"] * bsz
141
- const inp_pos = bsz
142
- const inp_KQ_mask = context * bsz
143
- const inp_K_shift = context
144
- const inp_sum = bsz
145
 
146
- return inp_tokens + inp_embd + inp_pos + inp_KQ_mask + inp_K_shift + inp_sum
147
- }
148
 
149
- function computeBuffer(context=8192, model_config, bsz=512) {
150
- if (bsz != 512) {
151
- alert("batch size other than 512 is currently not supported for the compute buffer, using batchsize 512 for compute buffer calculation, end result result will be an overestimatition")
152
- }
153
- return (context / 1024 * 2 + 0.75) * model_config["num_attention_heads"] * 1024 * 1024
154
  }
 
 
155
 
156
- function kvCache(context=8192, model_config, cache_bit=16) {
157
- const n_gqa = model_config["num_attention_heads"] / model_config["num_key_value_heads"]
158
- const n_embd_gqa = model_config["hidden_size"] / n_gqa
159
- const n_elements = n_embd_gqa * (model_config["num_hidden_layers"] * context)
160
- const size = 2 * n_elements
161
- return size * (cache_bit / 8)
162
- }
163
 
164
- function contextSize(context=8192, model_config, bsz=512, cache_bit=16) {
165
- return Number.parseFloat((inputBuffer(context, model_config, bsz) + kvCache(context, model_config, cache_bit) + computeBuffer(context, model_config, bsz)).toFixed(2))
166
- }
167
 
168
- function modelSize(model_config, bpw=4.5) {
169
- return Number.parseFloat((model_config["parameters"] * bpw / 8).toFixed(2))
170
- }
171
 
172
- async function calculateSizes(format) {
173
- try {
174
- const model_config = await modelConfig(document.getElementById("modelsearch").value.replace("https://huggingface.co/", ""), document.getElementById("hf_token").value)
175
- const context = parseInt(document.getElementById("contextsize").value)
176
- let bsz = 512
177
- let cache_bit = 16
178
- let bpw = 0
179
- if (format === "gguf") {
180
- bsz = parseInt(document.getElementById("batchsize").value)
181
- bpw = gguf_quants[document.getElementById("quantsize").innerText]
182
-
183
- } else if (format == "exl2") {
184
- cache_bit = Number.parseInt(document.getElementById("kvCache").value)
185
- bpw = Number.parseFloat(document.getElementById("bpw").value)
186
- }
187
-
188
- const model_size = modelSize(model_config, bpw)
189
- const context_size = contextSize(context, model_config, bsz, cache_bit)
190
- const total_size = ((model_size + context_size) / 2**30)
191
- document.getElementById("resultmodel").innerText = (model_size / 2**30).toFixed(2)
192
- document.getElementById("resultcontext").innerText = (context_size / 2**30).toFixed(2)
193
- const result_total_el = document.getElementById("resulttotal");
194
- result_total_el.innerText = total_size.toFixed(2)
195
-
196
- const gpu = document.getElementById("gpusearch").value
197
- if (gpu !== "") {
198
- const vram = parseFloat(gpu.split("-")[1].replace("GB", "").trim())
199
- if (vram - total_size > 0.5) {
200
- result_total_el.style.backgroundColor = "#bef264"
201
- } else if (vram - total_size > 0) {
202
- result_total_el.style.backgroundColor = "#facc15"
203
- } else {
204
- result_total_el.style.backgroundColor = "#ef4444"
205
- }
206
  }
207
- } catch(e) {
208
- alert(e);
209
  }
 
 
210
  }
211
- </script>
212
- <link href="./styles.css" rel="stylesheet">
213
- <title>Can I run it? - LLM VRAM Calculator</title>
214
- </head>
215
- <body class="p-8">
216
- <div x-data="{ format: 'gguf' }" class="flex flex-col max-h-screen items-center mt-16 gap-10">
217
- <h1 class="text-xl font-semibold leading-6 text-gray-900">
218
- LLM Model, Can I run it?
219
- </h1>
220
- <p>
221
- To support gated or private repos, you need to <a href="https://huggingface.co/settings/tokens" style="color: #4444ff"><b>create an authentification token</b></a>, to check the box <span style="color: #6e1818"><b>"Read access to contents of all public gated repos you can access"</b></span> and then enter the token in the field below.
222
- </p>
223
-
224
- <div class="flex flex-col gap-10">
225
- <div class="w-auto flex flex-col gap-4">
226
- <!-- Huggingface Authentification Token -->
227
- <div
228
- class="relative"
229
- x-data="{
 
 
 
230
  results: null,
231
  query: null
232
- }"
233
- >
234
- <label
235
- for="gpusearch"
236
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
237
- >Huggingface Token (optional)</label
238
- >
239
- <input
240
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
241
- id="hf_token"
242
- />
243
- </div>
244
- <!-- GPU Selector -->
245
- <div
246
- class="relative"
247
- x-data="{
248
  results: null,
249
  query: null
250
- }"
251
- >
252
- <label
253
- for="gpusearch"
254
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
255
- >GPU (optional)</label
256
- >
257
- <input
258
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
259
- placeholder="GeForce RTX 3090 - 24 GB"
260
- id="gpusearch"
261
- name="gpusearch"
262
- list="gpulist"
263
- x-model="query"
264
- @keypress.debounce.150ms="results = query === '' ? [] : formatGpu(tableToObj(strToHtml(await fetch('https://corsproxy.io/?https://www.techpowerup.com/gpu-specs/?ajaxsrch=' + query).then(r => r.text())).querySelector('table')))"
265
- />
266
- <datalist id="gpulist">
267
- <template x-for="item in results">
268
- <option :value="item" x-text="item"></option>
269
- </template>
270
- </datalist>
271
- </div>
272
- <!-- Model Selector -->
273
 
274
 
275
- <div class="flex flex-row gap-4 relative">
276
- <label
277
- for="contextsize"
278
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
279
- >
280
- Model (unquantized)
281
- </label>
282
- <div
283
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
284
- x-data="{
285
  open: false,
286
  value: 'Nexusflow/Starling-LM-7B-beta',
287
  results: null,
@@ -301,175 +294,99 @@
301
 
302
  focusAfter && focusAfter.focus()
303
  }
304
- }"
305
- x-on:keydown.escape.prevent.stop="close($refs.input)"
306
- x-id="['model-typeahead']"
307
- class="relative"
308
- >
309
- <!-- Input -->
310
- <input
311
- id="modelsearch"
312
- x-ref="input"
313
- x-on:click="toggle()"
314
- @keypress.debounce.150ms="results = (await
315
  fetch('https://huggingface.co/api/quicksearch?type=model&q=' +
316
  encodeURIComponent(value)).then(r => r.json())).models.filter(m => !m.id.includes('GGUF') && !m.id.includes('AWQ') && !m.id.includes('GPTQ') && !m.id.includes('exl2'));"
317
- :aria-expanded="open"
318
- :aria-controls="$id('model-typeahead')"
319
- x-model="value"
320
- class="flex justify-between items-center gap-2 w-full"
321
- />
322
 
323
- <!-- Panel -->
324
- <div
325
- x-ref="panel"
326
- x-show="open"
327
- x-transition.origin.top.left
328
- x-on:click.outside="close($refs.input)"
329
- :id="$id('model-typeahead')"
330
- style="display: none"
331
- class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"
332
- >
333
- <template x-for="result in results">
334
- <a
335
- @click="value = result.id; close($refs.input)"
336
- x-text="result.id"
337
- class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"
338
- ></a>
339
- </template>
340
- </div>
341
  </div>
342
  </div>
 
343
 
344
 
345
- <!-- Context Size Selector -->
346
- <div class="relative">
347
- <label
348
- for="contextsize"
349
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
350
- >
351
- Context Size
352
- </label>
353
- <input
354
- value="8192"
355
- type="number"
356
- name="contextsize"
357
- id="contextsize"
358
- step="1024"
359
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
360
- />
361
- </div>
362
- <!-- Quant Format Selector -->
363
- <div class="relative">
364
- <label
365
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
366
- >Quant Format</label
367
- >
368
- <fieldset
369
- x-model="format"
370
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
371
- >
372
- <legend class="sr-only">Quant format</legend>
373
- <div
374
- class="space-y-4 sm:flex sm:items-center sm:space-x-10 sm:space-y-0"
375
- >
376
- <div class="flex items-center">
377
- <input
378
- id="gguf-format"
379
- name="quant-format"
380
- type="radio"
381
- value="gguf"
382
- checked
383
- class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
384
- />
385
- <label
386
- for="gguf-format"
387
- class="ml-3 block text-sm font-medium leading-6 text-gray-900"
388
- >GGUF</label
389
- >
390
- </div>
391
- <div class="flex items-center">
392
- <input
393
- id="exl2-format"
394
- name="quant-format"
395
- type="radio"
396
- value="exl2"
397
- class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
398
- />
399
- <label
400
- for="exl2-format"
401
- class="ml-3 block text-sm font-medium leading-6 text-gray-900"
402
- >EXL2</label
403
- >
404
- </div>
405
- <div class="flex items-center">
406
- <input
407
- id="gptq-format"
408
- name="quant-format"
409
- type="radio"
410
- disabled
411
- value="gptq"
412
- class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
413
- />
414
- <label
415
- for="gptq-format"
416
- class="ml-3 block text-sm font-medium leading-6 text-gray-900"
417
- >GPTQ (coming soon)</label
418
- >
419
- </div>
420
  </div>
421
- </fieldset>
422
- </div>
423
- <!-- EXL2 Options -->
424
- <div x-show="format === 'exl2'" class="flex flex-row gap-4">
425
- <div class="relative flex-grow">
426
- <label
427
- for="bpw"
428
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
429
- >
430
- BPW
431
- </label>
432
- <input
433
- value="4.5"
434
- type="number"
435
- step="0.01"
436
- id="bpw"
437
- name="bpw"
438
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
439
- />
440
- </div>
441
- <div
442
- class="flex-shrink relative rounded-md"
443
- >
444
- <div
445
- class="w-fit p-3 h-full flex items-center gap-2 justify-center rounded-md border-0 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
446
- >
447
- <label
448
- for="kvCache"
449
- class="inline-block bg-white text-xs font-medium text-gray-900"
450
- >
451
- KV Cache
452
- </label>
453
- <select id="kvCache" name="kvCache">
454
- <option value="16">16 bit</option>
455
- <option value="8">8 bit</option>
456
- <option value="4">4 bit</option>
457
- </select>
458
  </div>
459
  </div>
 
 
 
 
 
 
 
 
 
 
 
460
  </div>
461
- <!-- GGUF Options -->
462
- <div x-show="format === 'gguf'" class="relative">
463
- <div class="flex flex-row gap-4">
464
- <label
465
- for="contextsize"
466
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
467
- >
468
- Quantization Size
469
  </label>
470
- <div
471
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
472
- x-data="{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
473
  open: false,
474
  value: '',
475
  toggle() {
@@ -488,42 +405,24 @@
488
 
489
  focusAfter && focusAfter.focus()
490
  }
491
- }"
492
- x-on:keydown.escape.prevent.stop="close($refs.button)"
493
- x-id="['dropdown-button']"
494
- class="relative"
495
- >
496
- <!-- Button -->
497
- <button
498
- x-ref="button"
499
- x-on:click="toggle()"
500
- :aria-expanded="open"
501
- :aria-controls="$id('dropdown-button')"
502
- type="button"
503
- id="quantsize"
504
- x-text="value.length === 0 ? 'Q4_K_S' : value"
505
- class="flex justify-between items-center gap-2 w-full"
506
- >
507
- Q4_K_S
508
 
509
- <!-- Heroicon: chevron-down -->
510
- <svg
511
- xmlns="http://www.w3.org/2000/svg"
512
- class="h-5 w-5 text-gray-400"
513
- viewBox="0 0 20 20"
514
- fill="currentColor"
515
- >
516
- <path
517
- fill-rule="evenodd"
518
- d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1 0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z"
519
- clip-rule="evenodd"
520
- />
521
- </svg>
522
- </button>
523
 
524
- <!-- Panel -->
525
- <div
526
- x-data="{ quants: [
527
  'IQ1_S',
528
  'IQ2_XXS',
529
  'IQ2_XS',
@@ -532,6 +431,7 @@
532
  'IQ3_XXS',
533
  'IQ3_XS',
534
  'Q2_K',
 
535
  'Q3_K_S',
536
  'IQ3_S',
537
  'IQ3_M',
@@ -547,91 +447,63 @@
547
  'Q5_K_M',
548
  'Q6_K',
549
  'Q8_0'
550
- ]}"
551
- x-ref="panel"
552
- x-show="open"
553
- x-transition.origin.top.left
554
- x-on:click.outside="close($refs.button)"
555
- :id="$id('dropdown-button')"
556
- style="display: none"
557
- class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"
558
- >
559
- <template x-for="quant in quants">
560
- <a
561
- @click="value = quant; close($refs.button)"
562
- x-text="quant"
563
- class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"
564
- ></a>
565
- </template>
566
- </div>
567
- </div>
568
- <div class="relative">
569
- <label
570
- for="batchsize"
571
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
572
- >
573
- Batch Size
574
- </label>
575
- <input
576
- value="512"
577
- type="number"
578
- step="128"
579
- id="batchsize"
580
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
581
- />
582
  </div>
583
  </div>
 
 
 
 
 
 
 
 
584
  </div>
585
- <button
586
- type="button"
587
- class="rounded-md bg-slate-800 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-slate-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600"
588
- @click="calculateSizes(format)"
589
- >
590
- Submit
591
- </button>
592
  </div>
593
- <div class="w-auto flex flex-col gap-4">
594
- <div class="relative">
595
- <label
596
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
597
- >
598
- Model Size (GB)
599
- </label>
600
- <div
601
- id="resultmodel"
602
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
603
- >4.20</div>
604
- </div>
605
- <div class="relative">
606
- <label
607
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
608
- >
609
- Context Size (GB)
610
- </label>
611
- <div
612
- id="resultcontext"
613
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
614
- >6.90</div>
615
- </div>
616
- <div class="relative">
617
- <label
618
- class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
619
- >
620
- Total Size (GB)
621
- </label>
622
- <div
623
- id="resulttotal"
624
- class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
625
- >420.69</div>
626
- </div>
627
  </div>
628
  </div>
629
  </div>
630
- <script
631
- src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"
632
- ></script>
633
- <script defer>
634
- calculateSizes("gguf")
635
- </script>
636
- </body>
637
- </html>
 
1
  <!DOCTYPE html>
2
  <html lang="en">
3
+
4
+ <head>
5
+ <meta charset="UTF-8" />
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
+ <script>
8
+ function strToHtml(str) {
9
+ let parser = new DOMParser();
10
+ return parser.parseFromString(str, "text/html");
11
+ }
12
+
13
+ //Short, jQuery-independent function to read html table and write them into an Array.
14
+ //Kudos to RobG at StackOverflow
15
+ function tableToObj(table) {
16
+ var rows = table.rows;
17
+ var propCells = rows[0].cells;
18
+ var propNames = [];
19
+ var results = [];
20
+ var obj, row, cells;
21
+
22
+ // Use the first row for the property names
23
+ // Could use a header section but result is the same if
24
+ // there is only one header row
25
+ for (var i = 0, iLen = propCells.length; i < iLen; i++) {
26
+ propNames.push(
27
+ (propCells[i].textContent || propCells[i].innerText).trim()
28
+ );
29
  }
30
 
31
+ // Use the rows for data
32
+ // Could use tbody rows here to exclude header & footer
33
+ // but starting from 1 gives required result
34
+ for (var j = 1, jLen = rows.length; j < jLen; j++) {
35
+ cells = rows[j].cells;
36
+ obj = {};
 
 
37
 
38
+ for (var k = 0; k < iLen; k++) {
39
+ obj[propNames[k]] = (
40
+ cells[k].textContent || cells[k].innerText
41
+ ).trim();
 
 
 
42
  }
43
+ results.push(obj);
44
+ }
45
+ return results;
46
+ }
47
 
48
+ function formatGpu(gpus) {
49
+ return gpus.map(
50
+ (g) => `${g["Product Name"]} - ${g["Memory"].split(",")[0]}`
51
+ );
52
+ }
 
53
 
54
+ const gguf_quants = {
55
+ "IQ1_S": 1.56,
56
+ "IQ2_XXS": 2.06,
57
+ "IQ2_XS": 2.31,
58
+ "IQ2_S": 2.5,
59
+ "IQ2_M": 2.7,
60
+ "IQ3_XXS": 3.06,
61
+ "IQ3_XS": 3.3,
62
+ "Q2_K": 3.35,
63
+ "Q2_K_XL": 3.67, // Added
64
+ "Q3_K_S": 3.5,
65
+ "IQ3_S": 3.5,
66
+ "IQ3_M": 3.7,
67
+ "Q3_K_M": 3.91,
68
+ "Q3_K_L": 4.27,
69
+ "IQ4_XS": 4.25,
70
+ "IQ4_NL": 4.5,
71
+ "Q4_0": 4.55,
72
+ "Q4_K_S": 4.58,
73
+ "Q4_K_M": 4.85,
74
+ "Q5_0": 5.54,
75
+ "Q5_K_S": 5.54,
76
+ "Q5_K_M": 5.69,
77
+ "Q6_K": 6.59,
78
+ "Q8_0": 8.5,
79
+ }
80
+
81
+ async function modelConfig(hf_model, hf_token) {
82
+ auth = hf_token == "" ? {} : {
83
+ headers: {
84
+ 'Authorization': `Bearer ${hf_token}`
85
  }
 
86
  }
87
+ let config_res = await fetch(
88
+ `https://huggingface.co/${hf_model}/raw/main/config.json`, auth
89
+ )
90
+ if (config_res.status === 401) {
91
+ throw new Error("Model is either private or gated, you must provide an access token")
92
  }
93
+ if (config_res.status === 403) {
94
+ throw new Error("Model is either private or gated and provided access token does not have access to the repo")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  }
96
+ let config = await config_res.json()
97
+ let model_size = 0
98
+ //find correct variable for dtype, some models have it in config.torch_dtype, some in config.text_config.dtype
99
+ let dtypeValue = config.torch_dtype ?? config.text_config?.dtype ?? config.text_config?.torch_dtype;
100
+ if (dtypeValue === undefined) {
101
+ throw new Error("dtype not found in config.torch_dtype or config.text_config.dtype");
102
+ }
103
+ let dtype = parseInt(dtypeValue.replaceAll(/\D/g, '')) / 8;
104
+ if ("text_config" in config) {
105
+ config = config["text_config"]
106
+ }
107
+ try {
108
+ model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/model.safetensors.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype
109
+ if (isNaN(model_size)) {
110
+ throw new Error("no size in safetensors metadata")
 
 
 
 
 
 
111
  }
112
+ } catch (e) {
113
  try {
114
+ model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/pytorch_model.bin.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype
115
  if (isNaN(model_size)) {
116
+ throw new Error("no size in pytorch metadata")
117
  }
118
+ } catch {
119
  try {
120
+ let modelCardInfos = await fetch(`https://huggingface.co/api/models/${hf_model}`)
121
+ .then(r => r.json());
122
+ model_size = ((modelCardInfos.safetensors?.total) ?? (modelCardInfos.gguf?.total))
123
  if (isNaN(model_size)) {
124
  throw new Error("no size in pytorch metadata")
125
  }
126
  } catch {
127
+ throw new Error("Couldn't determine model size from safetensor/pytorch index metadata nor from the model card. If the model is an unsharded pytorch model, it is not supported by this calculator.")
 
 
 
 
 
 
 
 
 
128
  }
129
  }
 
 
130
  }
131
+ config.parameters = model_size
132
+ return config
133
+ }
134
 
135
+ function inputBuffer(context = 8192, model_config, bsz = 512) {
136
+ /* Calculation taken from github:ggerganov/llama.cpp/llama.cpp:11248
137
+ ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
138
+ ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);
139
+ ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
140
+ ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);
141
+ ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);
142
+ ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch);
143
 
144
+ n_embd is hidden size (github:ggeranov/llama.cpp/convert.py:248)
145
+ */
146
+ const inp_tokens = bsz
147
+ const inp_embd = model_config["hidden_size"] * bsz
148
+ const inp_pos = bsz
149
+ const inp_KQ_mask = context * bsz
150
+ const inp_K_shift = context
151
+ const inp_sum = bsz
152
 
153
+ return inp_tokens + inp_embd + inp_pos + inp_KQ_mask + inp_K_shift + inp_sum
154
+ }
155
 
156
+ function computeBuffer(context = 8192, model_config, bsz = 512) {
157
+ if (bsz != 512) {
158
+ alert("batch size other than 512 is currently not supported for the compute buffer, using batchsize 512 for compute buffer calculation, end result result will be an overestimatition")
 
 
159
  }
160
+ return (context / 1024 * 2 + 0.75) * model_config["num_attention_heads"] * 1024 * 1024
161
+ }
162
 
163
+ function kvCache(context = 8192, model_config, cache_bit = 16) {
164
+ const n_gqa = model_config["num_attention_heads"] / model_config["num_key_value_heads"]
165
+ const n_embd_gqa = model_config["hidden_size"] / n_gqa
166
+ const n_elements = n_embd_gqa * (model_config["num_hidden_layers"] * context)
167
+ const size = 2 * n_elements
168
+ return size * (cache_bit / 8)
169
+ }
170
 
171
+ function contextSize(context = 8192, model_config, bsz = 512, cache_bit = 16) {
172
+ return Number.parseFloat((inputBuffer(context, model_config, bsz) + kvCache(context, model_config, cache_bit) + computeBuffer(context, model_config, bsz)).toFixed(2))
173
+ }
174
 
175
+ function modelSize(model_config, bpw = 4.5) {
176
+ return Number.parseFloat((model_config["parameters"] * bpw / 8).toFixed(2))
177
+ }
178
 
179
+ async function calculateSizes(format) {
180
+ try {
181
+ const model_config = await modelConfig(document.getElementById("modelsearch").value.replace("https://huggingface.co/", ""), document.getElementById("hf_token").value)
182
+ const context = parseInt(document.getElementById("contextsize").value)
183
+ let bsz = 512
184
+ let cache_bit = 16
185
+ let bpw = 0
186
+ if (format === "gguf") {
187
+ bsz = parseInt(document.getElementById("batchsize").value)
188
+ bpw = gguf_quants[document.getElementById("quantsize").innerText]
189
+
190
+ } else if (format == "exl2") {
191
+ cache_bit = Number.parseInt(document.getElementById("kvCache").value)
192
+ bpw = Number.parseFloat(document.getElementById("bpw").value)
193
+ }
194
+
195
+ const model_size = modelSize(model_config, bpw)
196
+ const context_size = contextSize(context, model_config, bsz, cache_bit)
197
+ const total_size = ((model_size + context_size) / 2 ** 30)
198
+ document.getElementById("resultmodel").innerText = (model_size / 2 ** 30).toFixed(2)
199
+ document.getElementById("resultcontext").innerText = (context_size / 2 ** 30).toFixed(2)
200
+ const result_total_el = document.getElementById("resulttotal");
201
+ result_total_el.innerText = total_size.toFixed(2)
202
+
203
+ const gpu = document.getElementById("gpusearch").value
204
+ if (gpu !== "") {
205
+ const vram = parseFloat(gpu.split("-")[1].replace("GB", "").trim())
206
+ if (vram - total_size > 0.5) {
207
+ result_total_el.style.backgroundColor = "#bef264"
208
+ } else if (vram - total_size > 0) {
209
+ result_total_el.style.backgroundColor = "#facc15"
210
+ } else {
211
+ result_total_el.style.backgroundColor = "#ef4444"
 
212
  }
 
 
213
  }
214
+ } catch (e) {
215
+ alert(e);
216
  }
217
+ }
218
+ </script>
219
+ <link href="./styles.css" rel="stylesheet">
220
+ <title>Can I run it? - LLM VRAM Calculator</title>
221
+ </head>
222
+
223
+ <body class="p-8">
224
+ <div x-data="{ format: 'gguf' }" class="flex flex-col max-h-screen items-center mt-16 gap-10">
225
+ <h1 class="text-xl font-semibold leading-6 text-gray-900">
226
+ LLM Model, Can I run it?
227
+ </h1>
228
+ <p>
229
+ To support gated or private repos, you need to <a href="https://huggingface.co/settings/tokens"
230
+ style="color: #4444ff"><b>create an authentification token</b></a>, to check the box <span
231
+ style="color: #6e1818"><b>"Read access to contents of all public gated repos you can access"</b></span> and then
232
+ enter the token in the field below.
233
+ </p>
234
+
235
+ <div class="flex flex-col gap-10">
236
+ <div class="w-auto flex flex-col gap-4">
237
+ <!-- Huggingface Authentification Token -->
238
+ <div class="relative" x-data="{
239
  results: null,
240
  query: null
241
+ }">
242
+ <label for="gpusearch"
243
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">Huggingface
244
+ Token (optional)</label>
245
+ <input
246
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
247
+ id="hf_token" />
248
+ </div>
249
+ <!-- GPU Selector -->
250
+ <div class="relative" x-data="{
 
 
 
 
 
 
251
  results: null,
252
  query: null
253
+ }">
254
+ <label for="gpusearch"
255
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">GPU
256
+ (optional)</label>
257
+ <input
258
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
259
+ placeholder="GeForce RTX 3090 - 24 GB" id="gpusearch" name="gpusearch" list="gpulist" x-model="query"
260
+ @keypress.debounce.150ms="results = query === '' ? [] : formatGpu(tableToObj(strToHtml(await fetch('https://corsproxy.io/?https://www.techpowerup.com/gpu-specs/?ajaxsrch=' + query).then(r => r.text())).querySelector('table')))" />
261
+ <datalist id="gpulist">
262
+ <template x-for="item in results">
263
+ <option :value="item" x-text="item"></option>
264
+ </template>
265
+ </datalist>
266
+ </div>
267
+ <!-- Model Selector -->
 
 
 
 
 
 
 
 
268
 
269
 
270
+ <div class="flex flex-row gap-4 relative">
271
+ <label for="contextsize"
272
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
273
+ Model (unquantized)
274
+ </label>
275
+ <div
276
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
277
+ x-data="{
 
 
278
  open: false,
279
  value: 'Nexusflow/Starling-LM-7B-beta',
280
  results: null,
 
294
 
295
  focusAfter && focusAfter.focus()
296
  }
297
+ }" x-on:keydown.escape.prevent.stop="close($refs.input)" x-id="['model-typeahead']"
298
+ class="relative">
299
+ <!-- Input -->
300
+ <input id="modelsearch" x-ref="input" x-on:click="toggle()"
301
+ @keypress.debounce.150ms="results = (await
 
 
 
 
 
 
302
  fetch('https://huggingface.co/api/quicksearch?type=model&q=' +
303
  encodeURIComponent(value)).then(r => r.json())).models.filter(m => !m.id.includes('GGUF') && !m.id.includes('AWQ') && !m.id.includes('GPTQ') && !m.id.includes('exl2'));"
304
+ :aria-expanded="open" :aria-controls="$id('model-typeahead')" x-model="value"
305
+ class="flex justify-between items-center gap-2 w-full" />
 
 
 
306
 
307
+ <!-- Panel -->
308
+ <div x-ref="panel" x-show="open" x-transition.origin.top.left x-on:click.outside="close($refs.input)"
309
+ :id="$id('model-typeahead')" style="display: none"
310
+ class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10">
311
+ <template x-for="result in results">
312
+ <a @click="value = result.id; close($refs.input)" x-text="result.id"
313
+ class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"></a>
314
+ </template>
 
 
 
 
 
 
 
 
 
 
315
  </div>
316
  </div>
317
+ </div>
318
 
319
 
320
+ <!-- Context Size Selector -->
321
+ <div class="relative">
322
+ <label for="contextsize"
323
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
324
+ Context Size
325
+ </label>
326
+ <input value="8192" type="number" name="contextsize" id="contextsize" step="1024"
327
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" />
328
+ </div>
329
+ <!-- Quant Format Selector -->
330
+ <div class="relative">
331
+ <label class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">Quant
332
+ Format</label>
333
+ <fieldset x-model="format"
334
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6">
335
+ <legend class="sr-only">Quant format</legend>
336
+ <div class="space-y-4 sm:flex sm:items-center sm:space-x-10 sm:space-y-0">
337
+ <div class="flex items-center">
338
+ <input id="gguf-format" name="quant-format" type="radio" value="gguf" checked
339
+ class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600" />
340
+ <label for="gguf-format" class="ml-3 block text-sm font-medium leading-6 text-gray-900">GGUF</label>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
341
  </div>
342
+ <div class="flex items-center">
343
+ <input id="exl2-format" name="quant-format" type="radio" value="exl2"
344
+ class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600" />
345
+ <label for="exl2-format" class="ml-3 block text-sm font-medium leading-6 text-gray-900">EXL2</label>
346
+ </div>
347
+ <div class="flex items-center">
348
+ <input id="gptq-format" name="quant-format" type="radio" disabled value="gptq"
349
+ class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600" />
350
+ <label for="gptq-format" class="ml-3 block text-sm font-medium leading-6 text-gray-900">GPTQ (coming
351
+ soon)</label>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
  </div>
353
  </div>
354
+ </fieldset>
355
+ </div>
356
+ <!-- EXL2 Options -->
357
+ <div x-show="format === 'exl2'" class="flex flex-row gap-4">
358
+ <div class="relative flex-grow">
359
+ <label for="bpw"
360
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
361
+ BPW
362
+ </label>
363
+ <input value="4.5" type="number" step="0.01" id="bpw" name="bpw"
364
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" />
365
  </div>
366
+ <div class="flex-shrink relative rounded-md">
367
+ <div
368
+ class="w-fit p-3 h-full flex items-center gap-2 justify-center rounded-md border-0 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6">
369
+ <label for="kvCache" class="inline-block bg-white text-xs font-medium text-gray-900">
370
+ KV Cache
 
 
 
371
  </label>
372
+ <select id="kvCache" name="kvCache">
373
+ <option value="16">16 bit</option>
374
+ <option value="8">8 bit</option>
375
+ <option value="4">4 bit</option>
376
+ </select>
377
+ </div>
378
+ </div>
379
+ </div>
380
+ <!-- GGUF Options -->
381
+ <div x-show="format === 'gguf'" class="relative">
382
+ <div class="flex flex-row gap-4">
383
+ <label for="contextsize"
384
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
385
+ Quantization Size
386
+ </label>
387
+ <div
388
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
389
+ x-data="{
390
  open: false,
391
  value: '',
392
  toggle() {
 
405
 
406
  focusAfter && focusAfter.focus()
407
  }
408
+ }" x-on:keydown.escape.prevent.stop="close($refs.button)" x-id="['dropdown-button']" class="relative">
409
+ <!-- Button -->
410
+ <button x-ref="button" x-on:click="toggle()" :aria-expanded="open" :aria-controls="$id('dropdown-button')"
411
+ type="button" id="quantsize" x-text="value.length === 0 ? 'Q4_K_S' : value"
412
+ class="flex justify-between items-center gap-2 w-full">
413
+ Q4_K_S
 
 
 
 
 
 
 
 
 
 
 
414
 
415
+ <!-- Heroicon: chevron-down -->
416
+ <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 text-gray-400" viewBox="0 0 20 20"
417
+ fill="currentColor">
418
+ <path fill-rule="evenodd"
419
+ d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1 0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z"
420
+ clip-rule="evenodd" />
421
+ </svg>
422
+ </button>
 
 
 
 
 
 
423
 
424
+ <!-- Panel -->
425
+ <div x-data="{ quants: [
 
426
  'IQ1_S',
427
  'IQ2_XXS',
428
  'IQ2_XS',
 
431
  'IQ3_XXS',
432
  'IQ3_XS',
433
  'Q2_K',
434
+ 'Q2_K_XL',
435
  'Q3_K_S',
436
  'IQ3_S',
437
  'IQ3_M',
 
447
  'Q5_K_M',
448
  'Q6_K',
449
  'Q8_0'
450
+ ]}" x-ref="panel" x-show="open" x-transition.origin.top.left x-on:click.outside="close($refs.button)"
451
+ :id="$id('dropdown-button')" style="display: none"
452
+ class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10">
453
+ <template x-for="quant in quants">
454
+ <a @click="value = quant; close($refs.button)" x-text="quant"
455
+ class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"></a>
456
+ </template>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
  </div>
458
  </div>
459
+ <div class="relative">
460
+ <label for="batchsize"
461
+ class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
462
+ Batch Size
463
+ </label>
464
+ <input value="512" type="number" step="128" id="batchsize"
465
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" />
466
+ </div>
467
  </div>
 
 
 
 
 
 
 
468
  </div>
469
+ <button type="button"
470
+ class="rounded-md bg-slate-800 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-slate-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600"
471
+ @click="calculateSizes(format)">
472
+ Submit
473
+ </button>
474
+ </div>
475
+ <div class="w-auto flex flex-col gap-4">
476
+ <div class="relative">
477
+ <label class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
478
+ Model Size (GB)
479
+ </label>
480
+ <div id="resultmodel"
481
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6">
482
+ 4.20</div>
483
+ </div>
484
+ <div class="relative">
485
+ <label class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
486
+ Context Size (GB)
487
+ </label>
488
+ <div id="resultcontext"
489
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6">
490
+ 6.90</div>
491
+ </div>
492
+ <div class="relative">
493
+ <label class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900">
494
+ Total Size (GB)
495
+ </label>
496
+ <div id="resulttotal"
497
+ class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6">
498
+ 420.69</div>
 
 
 
 
499
  </div>
500
  </div>
501
  </div>
502
+ </div>
503
+ <script src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
504
+ <script defer>
505
+ calculateSizes("gguf")
506
+ </script>
507
+ </body>
508
+
509
+ </html>