Spaces:
Running
Running
| # -*- coding: utf-8 -*- | |
| """Per-request facts: the chip strip, the static speed table, the code snippet. | |
| Nothing here is the Space's own wall time. Speed and price are computed from | |
| the request's packed size against measured reference points (`backend.py`). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import pprint | |
| import textwrap | |
| from html import escape | |
| import backend as be | |
| def _chip(label: str, value: str, accent: bool = False, hint: str = "") -> str: | |
| css = "fd-chip accent" if accent else "fd-chip" | |
| title = f' title="{escape(hint)}"' if hint else "" | |
| return f'<span class="{css}"{title}><i>{escape(label)}</i><b>{escape(value)}</b></span>' | |
| def fmt_ms(ms: float, lang: str) -> str: | |
| ru = lang == "ru" | |
| if ms < 1000: | |
| whole = int(ms + 0.5) # half up: 24.5 ms reads 25 | |
| return f"{whole} мс" if ru else f"{whole} ms" | |
| sec = f"{ms / 1000:.2f}" | |
| return (sec.replace(".", ",") + " с") if ru else sec + " s" | |
| def fmt_int(n: int, lang: str) -> str: | |
| return f"{n:,}".replace(",", " " if lang == "ru" else ",") | |
| def render_usage(cost: be.Cost, rate: float, api_rate: float, lang: str) -> str: | |
| """One strip of chips above the answer: facts of this one request.""" | |
| ru = lang == "ru" | |
| chips = [ | |
| _chip("модель" if ru else "model", "FRIDA-Decisions"), | |
| _chip("вопросов" if ru else "questions", str(cost.questions)), | |
| _chip("кандидатов" if ru else "candidates", str(cost.candidates)), | |
| _chip("токенов энкодера" if ru else "encoder tokens", fmt_int(cost.packed_tokens, lang), | |
| hint=(f"против {cost.naive_tokens} при раскладке «последовательность на вариант»" | |
| if ru else f"against {cost.naive_tokens} with one sequence per option")), | |
| _chip("выходных токенов" if ru else "output tokens", "0"), | |
| _chip("GPU" if ru else "GPU", "≈ " + fmt_ms(cost.gpu_ms, lang), accent=True, | |
| hint="RTX 5060 Ti, bf16"), | |
| _chip("за 1000 решений" if ru else "per 1000 decisions", | |
| f"${cost.cost_per_1000(rate):.4f}", accent=True, | |
| hint=(f"GPU по ${rate:.2f}/ч — ${be.gpu_per_mtok(rate):.4f} за 1M токенов " | |
| f"энкодера" if ru else | |
| f"GPU at ${rate:.2f}/h — ${be.gpu_per_mtok(rate):.4f} per 1M encoder " | |
| f"tokens")), | |
| _chip("то же у TypeSafe Jev" if ru else "same at TypeSafe Jev", | |
| f"${cost.api_cost_per_1000(api_rate):.4f}", | |
| hint=(f"{fmt_int(cost.api_tokens, lang)} входных токенов по ${api_rate:g}/M" | |
| if ru else | |
| f"{fmt_int(cost.api_tokens, lang)} input tokens at ${api_rate:g}/M")), | |
| ] | |
| cut = "" | |
| if cost.state_truncated: | |
| cut = (f", модель читает первые {cost.state_tokens}" if ru | |
| else f", the model reads the first {cost.state_tokens}") | |
| detail = ((f"Текст {cost.state_tokens} токенов{cut} · {cost.questions} вопрос(ов) " | |
| f"одним запросом · упаковка сократила вход энкодера в " | |
| f"{cost.packing_ratio:.1f} раза") if ru else | |
| (f"Text {cost.state_tokens} tokens{cut} · {cost.questions} question(s) " | |
| f"in one request · packing cut the encoder input " | |
| f"{cost.packing_ratio:.1f}x")) | |
| return (f'<div class="fd-strip-meta">{"".join(chips)}</div>' | |
| f'<div class="fd-note">{escape(detail)}</div>') | |
| def render_speed(cost: be.Cost, rate: float, api_rate: float, lang: str) -> str: | |
| """The static per-hardware table for this request, plus its assumptions.""" | |
| ru = lang == "ru" | |
| rows = [ | |
| ("RTX 5060 Ti · bf16" + (" · один поток" if ru else " · single stream"), | |
| "≈ " + fmt_ms(cost.gpu_ms, lang), | |
| (f"замер: {fmt_ms(be.FLOOR_GPU_MS, lang)} (1 вопрос) и " | |
| f"{fmt_ms(be.REF_GPU_MS, lang)} (3 вопроса) на тексте 402 токена; " | |
| f"~{be.REF_GPU_REQ_PER_S} запросов/с в один поток") if ru else | |
| (f"measured: {fmt_ms(be.FLOOR_GPU_MS, lang)} (1 question) and " | |
| f"{fmt_ms(be.REF_GPU_MS, lang)} (3 questions) at a 402-token text; " | |
| f"~{be.REF_GPU_REQ_PER_S} requests/s single stream")), | |
| ("CPU · 6 " + ("потоков" if ru else "threads") + " · fp32", | |
| "≈ " + fmt_ms(cost.cpu_fp32_ms, lang), | |
| (f"замер: {fmt_ms(be.CPU_REF_FP32_MS, lang)} на тексте 384 токена " | |
| f"с 3 вопросами" if ru else | |
| f"measured: {fmt_ms(be.CPU_REF_FP32_MS, lang)} at a 384-token text " | |
| f"with 3 questions")), | |
| ("CPU · 6 " + ("потоков" if ru else "threads") + " · ONNX int8", | |
| "≈ " + fmt_ms(cost.cpu_onnx_ms, lang), | |
| (f"замер: {fmt_ms(be.CPU_REF_ONNX_MS, lang)} на том же запросе; в таблице — " | |
| f"fp32 ÷ {str(be.ONNX_SPEEDUP).replace('.', ',')} " | |
| f"(наблюдалось от {str(be.ONNX_SPEEDUP_OBSERVED[0]).replace('.', ',')} " | |
| f"до {str(be.ONNX_SPEEDUP_OBSERVED[1]).replace('.', ',')})" if ru else | |
| f"measured: {fmt_ms(be.CPU_REF_ONNX_MS, lang)} on the same request; the table " | |
| f"is fp32 / {be.ONNX_SPEEDUP} (observed {be.ONNX_SPEEDUP_OBSERVED[0]} to " | |
| f"{be.ONNX_SPEEDUP_OBSERVED[1]})")), | |
| ] | |
| head = (("железо", "этот запрос", "откуда") if ru else ("hardware", "this request", "basis")) | |
| body = "".join(f'<tr><td>{escape(a)}</td><td class="num">{escape(b)}</td>' | |
| f'<td class="ref">{escape(c)}</td></tr>' for a, b, c in rows) | |
| table = (f'<table class="fd-speed"><thead><tr>' | |
| + "".join(f"<th>{h}</th>" for h in head) | |
| + f"</tr></thead><tbody>{body}</tbody></table>") | |
| source = be.GPU_RATE_SOURCE[0 if ru else 1] | |
| fine = ([ | |
| f"Время посчитано по размеру запроса: {cost.packed_tokens} токенов энкодера " | |
| f"против эталонных {be.REF_PACKED_TOKENS} (текст 402 токена, 3 вопроса). " | |
| "Между двумя замеренными размерами — линейно, выше — пропорционально, ниже " | |
| "наименьшего замера не уменьшается.", | |
| f"Цена тысячи решений: GPU по ${rate:.2f}/ч ({source}), пропускная способность " | |
| f"{be.REF_GPU_REQ_PER_S} запросов/с на эталонном запросе с RTX 5060 Ti, " | |
| "масштабированная по размеру; решение — один вопрос. В пересчёте на токены " | |
| f"это ${be.gpu_per_mtok(rate):.4f} за 1M токенов энкодера против ${api_rate:g} " | |
| "за 1M у API. Оценка предполагает полную загрузку GPU и пропускную способность " | |
| "арендуемого GPU на уровне RTX 5060 Ti.", | |
| f"TypeSafe Jev: ${api_rate:g} за 1M входных токенов, выход бесплатно. Сколько " | |
| "токенов он выставит за этот запрос, посчитано по его же счёту: за 735 запросов " | |
| f"бенчмарка ({be.TYPESAFE_BILL_SOURCE[0]}) счёт лёг на {be.TYPESAFE_CALL_TOKENS} + " | |
| f"{be.TYPESAFE_STATE_FACTOR} × токены текста + {be.TYPESAFE_CONTENT_FACTOR} × токены " | |
| "вопросов и вариантов — обвязка вызова плюс его собственная разметка.", | |
| ] if ru else [ | |
| f"Time is computed from the request size: {cost.packed_tokens} encoder tokens " | |
| f"against the reference {be.REF_PACKED_TOKENS} (a 402-token text, 3 questions). " | |
| "Linear between the two measured sizes, proportional above, never below the " | |
| "smallest measurement.", | |
| f"Cost per 1000 decisions: a GPU at ${rate:.2f}/h ({source}), throughput " | |
| f"{be.REF_GPU_REQ_PER_S} requests/s on the reference request on an RTX 5060 Ti, " | |
| "scaled by size; a decision is one question. Per token that is " | |
| f"${be.gpu_per_mtok(rate):.4f} per 1M encoder tokens against ${api_rate:g} per 1M " | |
| "billed by the API. The estimate assumes a fully loaded GPU and RTX 5060 " | |
| "Ti-class throughput on the rented GPU.", | |
| f"TypeSafe Jev: ${api_rate:g} per 1M input tokens, output free. How many tokens it " | |
| "bills for this request comes from its own invoice " | |
| f"({be.TYPESAFE_BILL_SOURCE[1]}): the bill fits {be.TYPESAFE_CALL_TOKENS} + " | |
| f"{be.TYPESAFE_STATE_FACTOR} × text tokens + {be.TYPESAFE_CONTENT_FACTOR} × question " | |
| "and option tokens — the wrapper around the call plus its own markup.", | |
| ]) | |
| return table + '<div class="fd-fine">' + "".join(f"<p>{escape(p)}</p>" for p in fine) + "</div>" | |
| def _json_is_python(value) -> bool: | |
| """Whether the JSON spelling of a value is also valid Python (no true/false/null).""" | |
| if isinstance(value, bool) or value is None: | |
| return False | |
| if isinstance(value, dict): | |
| return all(_json_is_python(k) and _json_is_python(v) for k, v in value.items()) | |
| if isinstance(value, (list, tuple)): | |
| return all(_json_is_python(v) for v in value) | |
| return True | |
| def render_python(payload: dict, backend_kind: str = "onnx") -> str: | |
| """The same request as working code on the released package.""" | |
| if _json_is_python(payload): | |
| body = json.dumps(payload, ensure_ascii=False, indent=4) | |
| else: | |
| body = pprint.pformat(payload, width=74, sort_dicts=False, indent=4) | |
| body = textwrap.indent(body, " ").lstrip() | |
| if backend_kind == "torch": | |
| load = ('from frida_decisions import Judge\n\n' | |
| 'judge = Judge.from_pretrained("ai-forever/FRIDA-Decisions") # GPU if available') | |
| else: | |
| load = ('from frida_decisions import OnnxJudge # PyTorch / GPU: Judge\n\n' | |
| 'judge = OnnxJudge.from_pretrained("ai-forever/FRIDA-Decisions")') | |
| return f'''{load} | |
| response = judge({body}) | |
| response["answers"] | |
| ''' | |