tokenizator / index.html
HubertRNA's picture
Upload index.html
27d6c61 verified
Raw History Blame Contribute Delete
39.6 kB
<!doctype html>
<html lang="pl">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Tokenizator - ReceptaNaAI.pl</title>
<meta name="description" content="Zobacz, jak modele językowe dzielą tekst na tokeny. Działa w przeglądarce, tekst nigdzie nie jest wysyłany.">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Montserrat:wght@600;700;800&family=Atkinson+Hyperlegible:wght@400;700&family=JetBrains+Mono:wght@500;700&display=swap" rel="stylesheet">
<style>
:root {
color-scheme: light;
--navy: #12304F;
--navy-2: #1B4F7A;
--text: #1C2A3A;
--muted: #4B5D70;
--cream: #FBF4DD;
--cream-hi: #FFF8DC;
--gold-dark: #8A6420;
--gold-mid: #C99A2E;
--gold-light: #F3D98B;
--gold: linear-gradient(135deg, #8A6420 0%, #D9AE45 22%, #FFF1B8 45%, #D4A93E 68%, #8A6420 100%);
--coral-d: #B3361F;
--font-head: "Montserrat", "Segoe UI", system-ui, sans-serif;
--font-body: "Atkinson Hyperlegible", "Segoe UI", system-ui, sans-serif;
--font-mono: "JetBrains Mono", ui-monospace, "Cascadia Code", Consolas, monospace;
--gloss: linear-gradient(180deg, rgba(255,255,255,.55) 0, rgba(255,255,255,0) 55%);
}
* { box-sizing: border-box; }
html { -webkit-text-size-adjust: 100%; }
body {
margin: 0;
background: linear-gradient(180deg, #E9F3EE 0, #F4EBD3 100%);
color: var(--text);
font-family: var(--font-body);
font-size: 1.0625rem;
line-height: 1.55;
}
a { color: var(--navy-2); text-underline-offset: 3px; }
a:hover { color: var(--coral-d); }
:focus-visible { outline: 3px solid var(--navy); outline-offset: 3px; }
.wrap { max-width: 1000px; margin: 0 auto; padding: 0 16px; }
.sr { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; }
/* ================= hero: niebo, miasto, kopuła ================= */
.hero {
position: relative; overflow: hidden; text-align: center; padding: 30px 0 150px;
background:
radial-gradient(ellipse 190px 42px at 10% 18%, rgba(255,255,255,.85), transparent 70%),
radial-gradient(ellipse 240px 50px at 88% 30%, rgba(255,255,255,.8), transparent 70%),
radial-gradient(ellipse 150px 34px at 68% 9%, rgba(255,255,255,.7), transparent 70%),
linear-gradient(180deg, #86C5E0 0%, #B9E0EC 48%, #E6F2EE 100%);
border-bottom: 5px solid var(--gold-mid);
}
.hero > svg { position: absolute; pointer-events: none; }
.hero .dome { inset: 0; width: 100%; height: 100%; }
.hero .skyline { left: 0; bottom: 0; width: 100%; height: 190px; }
.hero .plant { bottom: -14px; width: 230px; height: 210px; }
.hero .plant-l { left: -26px; }
.hero .plant-r { right: -26px; transform: scaleX(-1); }
.hero-inner { position: relative; z-index: 1; }
.brand { display: block; font-family: var(--font-head); font-weight: 700; color: var(--navy); text-decoration: none; letter-spacing: .02em; }
.brand:hover { color: var(--coral-d); }
.sign {
position: relative; display: table; margin: 30px auto 0;
padding: 14px clamp(22px, 5vw, 58px) 12px;
border: 5px solid transparent; border-radius: 18px;
background:
radial-gradient(circle at 15px 15px, #A87A24 0 3px, transparent 3.8px) padding-box,
radial-gradient(circle at calc(100% - 15px) 15px, #A87A24 0 3px, transparent 3.8px) padding-box,
radial-gradient(circle at 15px calc(100% - 15px), #A87A24 0 3px, transparent 3.8px) padding-box,
radial-gradient(circle at calc(100% - 15px) calc(100% - 15px), #A87A24 0 3px, transparent 3.8px) padding-box,
linear-gradient(var(--cream), var(--cream)) padding-box,
var(--gold) border-box;
box-shadow: 0 10px 24px rgba(18,48,79,.28);
}
.sign::before { content: ""; position: absolute; inset: 6px; border: 1.5px solid var(--gold-mid); border-radius: 11px; pointer-events: none; }
.sign .star { position: absolute; left: 50%; top: -21px; width: 28px; height: 28px; transform: translateX(-50%); }
h1 {
margin: 0; font-family: var(--font-head); font-weight: 800; text-transform: uppercase;
font-size: clamp(1.9rem, 7.2vw, 3.7rem); letter-spacing: .03em; line-height: 1.15; color: var(--navy);
}
.hero-sub { margin: 22px 0 0; font-family: var(--font-head); font-weight: 700; font-size: clamp(1.05rem, 3vw, 1.35rem); color: var(--navy); }
.lead { max-width: 58ch; margin: 8px auto 0; }
.privacy {
display: flex; align-items: center; gap: 14px; text-align: left;
max-width: 700px; margin: 22px auto 0; padding: 12px 18px 12px 12px;
border: 3px solid transparent; border-radius: 16px;
background: linear-gradient(rgba(251,244,221,.94), rgba(251,244,221,.94)) padding-box, var(--gold) border-box;
box-shadow: 0 6px 16px rgba(18,48,79,.18);
}
.privacy .lock {
flex: none; width: 48px; height: 48px; border-radius: 50%; display: grid; place-items: center;
border: 3px solid transparent;
background: radial-gradient(circle at 35% 28%, #2D6A9E, #12304F 70%) padding-box, var(--gold) border-box;
}
.privacy p { margin: 0; }
/* ================= panele ================= */
main { padding-bottom: 46px; }
.panel {
position: relative; margin-top: 54px; padding: 40px 22px 24px; border-radius: 22px;
background: rgba(255,255,255,.6);
-webkit-backdrop-filter: blur(8px); backdrop-filter: blur(8px);
box-shadow: 0 12px 30px rgba(18,48,79,.13);
}
.panel::before {
content: ""; position: absolute; inset: 0; border-radius: inherit; padding: 3px; background: var(--gold); pointer-events: none;
-webkit-mask: linear-gradient(#000 0 0) content-box, linear-gradient(#000 0 0);
-webkit-mask-composite: xor;
mask: linear-gradient(#000 0 0) content-box, linear-gradient(#000 0 0);
mask-composite: exclude;
}
.plaque {
position: absolute; z-index: 2; top: 0; left: 50%; transform: translate(-50%, -50%); margin: 0; white-space: nowrap;
padding: 7px 22px; border: 3px solid transparent; border-radius: 14px;
background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box;
font-family: var(--font-head); font-weight: 800; font-size: clamp(.86rem, 2.6vw, 1.08rem);
letter-spacing: .04em; text-transform: uppercase; color: var(--navy); line-height: 1.2;
box-shadow: 0 4px 10px rgba(18,48,79,.2);
}
.hint { color: var(--muted); margin: 0 0 14px; max-width: 64ch; }
/* ================= kafelki tokenów ================= */
.tok {
display: inline-block; position: relative; border: 2px solid var(--navy); border-radius: 8px;
padding: 1px 7px 2px; font-family: var(--font-mono); font-weight: 500; font-size: 1.05rem; line-height: 1.45;
white-space: pre-wrap; word-break: break-all; color: var(--navy);
background-color: var(--fill, #B3E2E0); background-image: var(--gloss);
box-shadow: 0 2px 0 rgba(18,48,79,.35);
}
.c0 { --fill: #B3E2E0; } .c1 { --fill: #F7B5A5; } .c2 { --fill: #F8DC8E; } .c3 { --fill: #C0CDEC; } .c4 { --fill: #C6E5BE; }
.tok.frag { background-image: repeating-linear-gradient(45deg, rgba(18,48,79,.24) 0 3px, transparent 3px 7px), var(--gloss); }
.sp, .nl { opacity: .45; }
.tid { display: block; font-size: .72rem; font-weight: 700; line-height: 1.1; text-align: center; color: var(--navy-2); margin-top: 1px; }
/* ================= wybór modelu ================= */
.chips { display: flex; flex-wrap: wrap; justify-content: center; gap: 12px; }
.chip { position: relative; }
.chip input { position: absolute; opacity: 0; inset: 0; width: 100%; height: 100%; margin: 0; cursor: pointer; }
.chip span {
display: inline-flex; align-items: center; gap: 8px; min-height: 46px; padding: 6px 22px;
border: 3px solid transparent; border-radius: 999px;
background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box;
font-family: var(--font-head); font-weight: 700; font-size: .95rem; color: var(--navy);
box-shadow: 0 3px 0 rgba(138,100,32,.55); transition: transform .08s, box-shadow .08s;
}
.chip input:hover + span { background: linear-gradient(var(--cream-hi), var(--cream-hi)) padding-box, var(--gold) border-box; }
.chip input:checked + span {
background: radial-gradient(circle at 35% 25%, #2B6395, #12304F 70%) padding-box, var(--gold) border-box;
color: var(--gold-light); box-shadow: 0 1px 0 rgba(138,100,32,.55); transform: translateY(2px);
}
.chip input:focus-visible + span { outline: 3px solid var(--navy); outline-offset: 3px; }
.chip small { font-family: var(--font-body); font-weight: 400; font-size: .8rem; }
.model-note { min-height: 1.6em; margin: 14px 0 0; text-align: center; color: var(--muted); font-size: .95rem; }
/* ================= wpisywanie ================= */
label.field { display: block; font-weight: 700; margin-bottom: 8px; }
textarea {
width: 100%; min-height: 130px; resize: vertical; padding: 12px 14px;
font-family: var(--font-mono); font-size: 1.05rem; line-height: 1.5; color: var(--navy);
border: 3px solid transparent; border-radius: 16px;
background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box;
box-shadow: inset 0 2px 8px rgba(18,48,79,.12);
}
textarea::placeholder { color: #6a7480; }
.examples { margin-top: 16px; }
.examples p { margin: 0 0 8px; font-weight: 700; }
.ex-list { display: flex; flex-wrap: wrap; gap: 8px; }
button { font: inherit; cursor: pointer; color: var(--navy); }
.btn-ex {
min-height: 40px; padding: 4px 16px; font-family: var(--font-head); font-weight: 700; font-size: .85rem;
border: 2px solid transparent; border-radius: 999px;
background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box;
}
.btn-ex:hover { background: linear-gradient(var(--cream-hi), var(--cream-hi)) padding-box, var(--gold) border-box; }
.btn {
min-height: 46px; padding: 6px 22px; font-family: var(--font-head); font-weight: 800; font-size: .95rem; letter-spacing: .02em;
border: 2px solid var(--gold-dark); border-radius: 999px; color: var(--navy);
background: linear-gradient(180deg, #FFF1B8 0%, #E3B94E 55%, #C99A2E 100%);
box-shadow: 0 3px 0 var(--gold-dark);
}
.btn:hover { background: linear-gradient(180deg, #FFF8D0 0%, #EBC55F 55%, #D6A93C 100%); }
.btn:active { transform: translateY(2px); box-shadow: 0 1px 0 var(--gold-dark); }
.btn[disabled] { opacity: .65; cursor: progress; }
/* ================= medale ze statystykami ================= */
.stats { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; margin-bottom: 20px; }
.stat { text-align: center; }
.coin {
display: grid; place-items: center; width: 112px; height: 112px; margin: 0 auto 8px; border-radius: 50%;
border: 5px solid transparent;
background: radial-gradient(circle at 35% 28%, #2D6A9E 0%, #164268 55%, #0C2742 100%) padding-box, var(--gold) border-box;
box-shadow: 0 6px 14px rgba(18,48,79,.3), inset 0 0 0 2px #0C2742, inset 0 0 0 3.5px rgba(243,217,139,.7);
}
.coin .v { font-family: var(--font-head); font-weight: 800; font-size: 1.85rem; line-height: 1; color: var(--gold-light); white-space: nowrap; }
.stat .k { display: block; font-size: .92rem; color: var(--muted); }
/* ================= wynik ================= */
.out-tools { display: flex; flex-wrap: wrap; justify-content: center; gap: 8px 24px; margin: 0 0 14px; }
.toggle { display: inline-flex; align-items: center; gap: 8px; font-weight: 700; cursor: pointer; min-height: 32px; }
.toggle input { width: 20px; height: 20px; accent-color: var(--navy); margin: 0; }
#tokens {
display: flex; flex-wrap: wrap; gap: 8px 6px; align-items: flex-start; min-height: 120px; padding: 16px;
background: #FBF7E8; border: 2px solid rgba(201,154,46,.75); border-radius: 16px;
box-shadow: inset 0 2px 8px rgba(18,48,79,.12);
}
#tokens .empty { color: var(--muted); margin: 0; align-self: center; }
#tokens .err { margin: 0; width: 100%; }
#tokens .err .btn { margin-top: 12px; display: block; }
.legend { margin: 12px 0 0; color: var(--muted); font-size: .95rem; max-width: 72ch; }
.legend .tok { font-size: .85rem; padding: 0 6px; box-shadow: none; vertical-align: 1px; }
@media (prefers-reduced-motion: no-preference) {
.first .tok { animation: pop .35s cubic-bezier(.3,1.6,.5,1) both; animation-delay: calc(var(--i, 0) * 28ms); }
@keyframes pop { from { transform: scale(.5) rotate(-4deg); opacity: 0; } to { transform: none; opacity: 1; } }
}
/* ================= porównanie ================= */
.compare-head { display: flex; flex-wrap: wrap; align-items: center; gap: 12px 18px; }
.progress { color: var(--muted); font-size: .95rem; }
#bars { list-style: none; margin: 18px 0 0; padding: 0; display: grid; gap: 10px; }
.bar-row { display: grid; grid-template-columns: 8.5rem 1fr; gap: 12px; align-items: center; }
.bar-row button { text-align: left; background: none; border: 0; padding: 4px 0; min-height: 40px; font-family: var(--font-head); font-weight: 700; font-size: .95rem; }
.bar-row button[aria-current="true"] { color: var(--coral-d); }
.bar-row button[aria-current="true"]::after { content: " (wybrany)"; font-family: var(--font-body); font-weight: 400; font-size: .8rem; color: var(--muted); }
.bar-track { display: flex; align-items: center; gap: 10px; }
.bar {
height: 32px; min-width: 6px; border: 2px solid var(--navy); border-radius: 999px;
background-color: var(--fill); background-image: var(--gloss); box-shadow: 0 2px 0 rgba(18,48,79,.35); transition: width .25s;
}
.bar-val { font-family: var(--font-mono); font-weight: 700; white-space: nowrap; color: var(--navy); }
.bar-val small { font-family: var(--font-body); font-weight: 400; color: var(--muted); }
.bar-na { color: var(--muted); }
.insight { margin: 16px 0 0; font-family: var(--font-head); font-weight: 700; font-size: .95rem; color: var(--navy); max-width: 70ch; }
/* ================= uwagi ================= */
.notes { display: grid; grid-template-columns: repeat(2, 1fr); gap: 0 34px; padding: 0; margin: 0; list-style: none; }
.notes li { padding: 14px 0; border-top: 2px solid rgba(201,154,46,.7); }
.notes b { font-family: var(--font-head); font-weight: 700; color: var(--navy); }
/* ================= stopka ================= */
footer { background: var(--navy); color: #DCE6F0; border-top: 5px solid var(--gold-mid); padding: 26px 0 36px; font-size: .92rem; }
footer p { margin: 0 0 8px; max-width: 90ch; }
footer strong { color: var(--gold-light); }
footer a { color: var(--gold-light); }
footer a:hover { color: #fff; }
@media (max-width: 900px) { .hero .plant { display: none; } }
@media (max-width: 720px) {
.hero { padding-bottom: 120px; }
.panel { padding: 38px 14px 20px; }
.stats { grid-template-columns: repeat(2, 1fr); row-gap: 14px; }
.coin { width: 96px; height: 96px; }
.coin .v { font-size: 1.55rem; }
.notes { grid-template-columns: 1fr; }
.bar-row { grid-template-columns: 1fr; gap: 0; }
.bar-row button { min-height: 36px; }
.privacy { text-align: left; }
}
</style>
</head>
<body>
<header class="hero">
<svg class="dome" viewBox="0 0 1000 100" preserveAspectRatio="none" aria-hidden="true">
<path d="M 4 100 V 26 A 496 22 0 0 1 996 26 V 100" fill="none" stroke="#C99A2E" stroke-width="5" vector-effect="non-scaling-stroke" opacity=".85"/>
<path d="M 14 100 V 30 A 486 21 0 0 1 986 30 V 100" fill="none" stroke="#FFF1B8" stroke-width="1.5" vector-effect="non-scaling-stroke" opacity=".9"/>
</svg>
<svg class="skyline" viewBox="0 0 1000 190" preserveAspectRatio="xMidYMax slice" aria-hidden="true">
<g fill="#D3EAEE" opacity=".9">
<rect x="60" y="110" width="40" height="80"/><rect x="150" y="90" width="34" height="100"/><rect x="330" y="120" width="46" height="70"/>
<rect x="560" y="105" width="44" height="85"/><rect x="760" y="95" width="38" height="95"/><rect x="900" y="115" width="50" height="75"/>
<path d="M200 190 A32 32 0 0 1 264 190 Z"/><path d="M820 190 A30 30 0 0 1 880 190 Z"/>
</g>
<g fill="#B5DCE4">
<rect x="296" y="46" width="6" height="144"/><ellipse cx="299" cy="56" rx="28" ry="7"/><rect x="297" y="26" width="4" height="22"/>
<path d="M690 190 L699 30 L708 190 Z"/><ellipse cx="699" cy="62" rx="20" ry="5"/>
<rect x="110" y="128" width="30" height="62"/><rect x="420" y="100" width="36" height="90"/><rect x="480" y="136" width="52" height="54"/>
<rect x="610" y="124" width="34" height="66"/><rect x="940" y="96" width="30" height="94"/>
<path d="M380 190 A44 44 0 0 1 468 190 Z" fill="#A4D3DD"/>
</g>
<g fill="#F3D98B" opacity=".9"><rect x="420" y="112" width="4" height="6"/><rect x="430" y="124" width="4" height="6"/><rect x="940" y="110" width="4" height="6"/><rect x="954" y="128" width="4" height="6"/><rect x="560" y="120" width="4" height="6"/></g>
</svg>
<svg class="plant plant-l" viewBox="0 0 230 210" aria-hidden="true">
<g stroke-linejoin="round">
<g transform="translate(78 210) rotate(-64) scale(.8)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(46) scale(.75)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(-34)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#2F7D55" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(22) scale(.95)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(-6) scale(1.15)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g>
</g>
</svg>
<svg class="plant plant-r" viewBox="0 0 230 210" aria-hidden="true">
<g stroke-linejoin="round">
<g transform="translate(78 210) rotate(-64) scale(.8)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(46) scale(.75)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(-34)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#2F7D55" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(22) scale(.95)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g>
<g transform="translate(78 210) rotate(-6) scale(1.15)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g>
</g>
</svg>
<div class="wrap hero-inner">
<a class="brand" href="https://receptanaai.pl" target="_blank" rel="noopener">ReceptaNaAI.pl</a>
<div class="sign">
<svg class="star" viewBox="0 0 24 24" aria-hidden="true"><defs><linearGradient id="gs" x1="0" y1="0" x2="1" y2="1"><stop offset="0" stop-color="#FFF1B8"/><stop offset="1" stop-color="#C99A2E"/></linearGradient></defs><path d="M12 0 C12.8 7 17 11.2 24 12 C17 12.8 12.8 17 12 24 C11.2 17 7 12.8 0 12 C7 11.2 11.2 7 12 0Z" fill="url(#gs)" stroke="#8A6420" stroke-width=".8"/></svg>
<h1>Tokenizator</h1>
</div>
<p class="hero-sub">Jak model językowy czyta Twój tekst?</p>
<p class="lead">Model nie widzi liter ani słów. Widzi tokeny: kawałki tekstu, na które tnie go tokenizer. Wpisz zdanie i zobacz, jak potnie je wybrany model.</p>
<div class="privacy">
<span class="lock" aria-hidden="true"><svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#F3D98B" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="5" y="11" width="14" height="10" rx="2" fill="#F3D98B" fill-opacity=".25"/><path d="M8 11V8a4 4 0 0 1 8 0v3"/></svg></span>
<p><strong>Tekst jest przetwarzany w Twojej przeglądarce i nigdzie nie jest wysyłany.</strong> Mimo to nie wpisuj tu danych pacjentów ani innych danych wrażliwych.</p>
</div>
</div>
</header>
<main class="wrap">
<section class="panel" aria-labelledby="lg-model">
<h2 class="plaque" id="lg-model">1. Wybierz model</h2>
<div class="chips" id="chips" role="radiogroup" aria-labelledby="lg-model"></div>
<p class="model-note" id="model-note" aria-live="polite"></p>
</section>
<section class="panel" aria-labelledby="h-text">
<h2 class="plaque" id="h-text">2. Wpisz tekst</h2>
<label class="field" for="text">Tekst do podzielenia na tokeny</label>
<textarea id="text" spellcheck="false" placeholder="Wpisz zdanie albo wybierz przykład poniżej"></textarea>
<div class="examples">
<p>Przykłady do porównania</p>
<div class="ex-list" id="examples"></div>
</div>
</section>
<section class="panel" aria-labelledby="h-out">
<h2 class="plaque" id="h-out">3. Zobacz tokeny</h2>
<div class="stats" id="stats" aria-live="polite">
<div class="stat"><span class="coin"><span class="v" id="s-tok">0</span></span><span class="k">tokenów</span></div>
<div class="stat"><span class="coin"><span class="v" id="s-chr">0</span></span><span class="k">znaków</span></div>
<div class="stat"><span class="coin"><span class="v" id="s-wrd">0</span></span><span class="k">słów</span></div>
<div class="stat"><span class="coin"><span class="v" id="s-tpw">-</span></span><span class="k">tokenów na słowo</span></div>
</div>
<div class="out-tools">
<label class="toggle"><input type="checkbox" id="opt-sp" checked> Pokaż spacje</label>
<label class="toggle"><input type="checkbox" id="opt-id"> Pokaż numery tokenów</label>
</div>
<div id="tokens" role="region" aria-label="Tokeny" aria-live="polite"></div>
<p class="legend" id="legend" hidden><span class="tok frag" aria-hidden="true">&#xFFFD;</span> Kratka oznacza fragment znaku zapisanego w kilku bajtach. Tak bywają cięte rzadsze litery, np. polskie znaki diakrytyczne.</p>
</section>
<section class="panel" aria-labelledby="h-cmp">
<h2 class="plaque" id="h-cmp">4. Porównaj modele</h2>
<p class="hint">Ten sam tekst, różne tokenizery. Porównanie pobiera pliki wszystkich tokenizerów, więc za pierwszym razem może chwilę potrwać.</p>
<div class="compare-head">
<button class="btn" id="btn-cmp" type="button">Porównaj wszystkie modele</button>
<span class="progress" id="cmp-progress" aria-live="polite"></span>
</div>
<ul id="bars" aria-label="Liczba tokenów w poszczególnych modelach"></ul>
<p class="insight" id="insight" aria-live="polite"></p>
</section>
<section class="panel" aria-labelledby="h-notes">
<h2 class="plaque" id="h-notes">Co warto zauważyć</h2>
<ul class="notes">
<li><b>Token to nie słowo.</b> Krótkie, częste słowa zwykle zajmują jeden token, a długie lub rzadkie dzielą się na kilka kawałków. Sprawdź to na nazwach leków.</li>
<li><b>Ten sam sens, inna liczba tokenów.</b> Porównaj zdanie po polsku z tym samym zdaniem po angielsku i zobacz, czy któryś model potrzebuje więcej tokenów.</li>
<li><b>Tokeny to waluta modeli.</b> Okno kontekstu i koszt zapytań w API liczy się w tokenach, a nie w znakach.</li>
<li><b>Każdy model ma własny tokenizer.</b> Liczba tokenów tego samego tekstu zależy od modelu, więc używaj porównania.</li>
</ul>
</section>
</main>
<footer>
<div class="wrap">
<p><strong>Skąd to się bierze.</strong> Tokenizacja działa lokalnie dzięki bibliotece Transformers.js (Hugging Face, licencja Apache 2.0). Przeglądarka pobiera bibliotekę z serwisu jsDelivr, pliki tokenizerów z Hugging Face Hub i czcionki z Google Fonts. Twój tekst nie jest do nich wysyłany.</p>
<p><strong>Licencje.</strong> Strona powstała z inspiracji projektem „The Tokenizer Playground” (Xenova, Hugging Face, Apache 2.0). Tokenizery pochodzą z repozytoriów społeczności Xenova i podlegają licencjom modeli, z których wywodzą się. Tokenizer oznaczony jako Claude jest nieoficjalny.</p>
<p><strong>Uwaga.</strong> Liczby tokenów służą celom edukacyjnym. Rzeczywiste zużycie tokenów w usługach komercyjnych może się różnić, np. przez znaczniki systemowe. Tu nie liczymy znaczników specjalnych.</p>
<p>© Recepta Na AI sp. z o.o. · <a href="https://receptanaai.pl" target="_blank" rel="noopener">ReceptaNaAI.pl</a></p>
</div>
</footer>
<script type="module">
/* ============================================================
Konfiguracja: modele (tokenizery z Hugging Face Hub)
Aby dodać model, dopisz wpis do tablicy MODELS.
============================================================ */
const LIB_URL = "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.1";
const MODELS = [
{ id: "Xenova/gpt-4", label: "GPT-4", note: "Tokenizer rodziny GPT-4 i GPT-3.5. Nowsze modele OpenAI mogą używać innych tokenizerów." },
{ id: "Xenova/llama-3-tokenizer", label: "Llama 3", note: "Tokenizer rodziny Llama 3 (Meta)." },
{ id: "Xenova/mistral-tokenizer-v3", label: "Mistral v3", note: "Tokenizer Mistral v3." },
{ id: "Xenova/gemma-tokenizer", label: "Gemma", note: "Tokenizer rodziny Gemma (Google)." },
{ id: "Xenova/claude-tokenizer", label: "Claude", badge: "nieoficjalny", note: "Tokenizer społecznościowy (Xenova), nieoficjalny. Liczby są orientacyjne i mogą różnić się od tych w faktycznych modelach Claude." },
];
const EXAMPLES = [
{ label: "Zdanie po polsku", text: "Pacjent z kardiomiopatią przerostową otrzymał bisoprolol i amiodaron." },
{ label: "To samo po angielsku", text: "The patient with hypertrophic cardiomyopathy received bisoprolol and amiodarone." },
{ label: "Skróty z kardiologii", text: "EKG: SR 72/min, LVEF 55%, NT-proBNP 340 pg/ml, hs-TnT 12 ng/l." },
{ label: "Nazwy leków", text: "metoprolol, rywaroksaban, atorwastatyna, ezetymib, sakubitryl/walsartan" },
{ label: "Łacina kliniczna", text: "Fibrillatio atriorum paroxysmalis, status post ablationem." },
{ label: "Zdanie bazowe", text: "Ala ma kota, a kot ma Alę." },
];
const MAX_TILES = 1500; // tyle kafelków maksymalnie rysujemy
const DEBOUNCE_MS = 150;
/* ============================================================
Pomocnicze
============================================================ */
const $ = (id) => document.getElementById(id);
const nf = new Intl.NumberFormat("pl-PL");
const nf1 = new Intl.NumberFormat("pl-PL", { minimumFractionDigits: 1, maximumFractionDigits: 1 });
const nf2 = new Intl.NumberFormat("pl-PL", { minimumFractionDigits: 2, maximumFractionDigits: 2 });
function plTokeny(n) {
if (n === 1) return "token";
const m10 = n % 10, m100 = n % 100;
return (m10 >= 2 && m10 <= 4 && !(m100 >= 12 && m100 <= 14)) ? "tokeny" : "tokenów";
}
const countChars = (s) => Array.from(s).length;
const countWords = (s) => s.trim() ? s.trim().split(/\s+/).length : 0;
const modelById = (id) => MODELS.find((m) => m.id === id);
/* ============================================================
Stan
============================================================ */
const params = new URLSearchParams(location.search);
let currentId = modelById(params.get("model")) ? params.get("model") : MODELS[0].id;
let runId = 0;
let firstRender = true;
let compareOn = false;
const loaded = new Map(); // id -> tokenizer (gotowy)
const loading = new Map(); // id -> Promise<tokenizer>
const failed = new Set(); // id, którym nie udało się załadować
let lib = null;
/* ============================================================
Ładowanie biblioteki i tokenizerów
============================================================ */
async function getLib() {
if (lib) return lib;
lib = await import(LIB_URL);
lib.env.allowLocalModels = false;
return lib;
}
function fixDecoder(tok) {
// Tokenizery typu Llama/Mistral kończą dekoder krokiem StripDecoder, który obcina wiodącą spację tokenu.
// Zdejmujemy go, żeby przy pojedynczych tokenach było widać, do którego tokenu należy spacja.
try {
const list = tok.decoder?.decoders;
if (Array.isArray(list) && list.at(-1)?.constructor?.name === "StripDecoder") list.pop();
} catch (_) { /* wygląd nie jest krytyczny */ }
return tok;
}
function getTokenizer(id) {
if (loaded.has(id)) return Promise.resolve(loaded.get(id));
if (loading.has(id)) return loading.get(id);
const p = (async () => {
const { AutoTokenizer } = await getLib();
const tok = fixDecoder(await AutoTokenizer.from_pretrained(id));
loaded.set(id, tok);
failed.delete(id);
return tok;
})();
loading.set(id, p);
p.catch(() => { failed.add(id); }).finally(() => loading.delete(id));
return p;
}
function encodeIds(tok, text) {
// Bez znaczników specjalnych (np. BOS), żeby liczby były porównywalne między modelami.
const ids = tok.encode(text, { add_special_tokens: false });
return Array.from(ids, Number);
}
/* ============================================================
Interfejs: wybór modelu
============================================================ */
function buildChips() {
const box = $("chips");
MODELS.forEach((m) => {
const lab = document.createElement("label");
lab.className = "chip";
const inp = document.createElement("input");
inp.type = "radio"; inp.name = "model"; inp.value = m.id; inp.checked = m.id === currentId;
inp.addEventListener("change", () => setModel(m.id));
const span = document.createElement("span");
span.append(m.label);
if (m.badge) { const s = document.createElement("small"); s.textContent = "(" + m.badge + ")"; span.append(s); }
lab.append(inp, span);
box.append(lab);
});
updateNote();
}
function updateNote() {
const m = modelById(currentId);
$("model-note").textContent = m ? m.note : "";
}
function setModel(id) {
currentId = id;
document.querySelectorAll('input[name="model"]').forEach((r) => { r.checked = r.value === id; });
updateNote();
render();
}
function buildExamples() {
const box = $("examples");
EXAMPLES.forEach((ex) => {
const b = document.createElement("button");
b.type = "button"; b.className = "btn-ex"; b.textContent = ex.label;
b.addEventListener("click", () => { $("text").value = ex.text; render(); });
box.append(b);
});
}
/* ============================================================
Rysowanie kafelków (tekst użytkownika wstawiamy wyłącznie przez textContent)
============================================================ */
function fillTileText(el, text, showSpaces) {
if (!showSpaces) { el.textContent = text; return; }
let buf = "";
const flush = () => { if (buf) { el.append(document.createTextNode(buf)); buf = ""; } };
for (const ch of text) {
if (ch === " ") { flush(); const s = document.createElement("span"); s.className = "sp"; s.textContent = "·"; el.append(s); }
else if (ch === "\n") { flush(); const s = document.createElement("span"); s.className = "nl"; s.textContent = "↵"; el.append(s); el.append(document.createTextNode("\n")); }
else if (ch === "\t") { flush(); const s = document.createElement("span"); s.className = "sp"; s.textContent = "⇥"; el.append(s); }
else buf += ch;
}
flush();
}
function drawTokens(ids, decoded) {
const box = $("tokens");
box.replaceChildren();
box.classList.toggle("first", firstRender);
const showSp = $("opt-sp").checked, showId = $("opt-id").checked;
const n = Math.min(ids.length, MAX_TILES);
let hasFrag = false;
const frag = document.createDocumentFragment();
for (let i = 0; i < n; i++) {
const t = document.createElement("span");
const bad = decoded[i].includes("�");
if (bad) hasFrag = true;
t.className = "tok c" + (i % 5) + (bad ? " frag" : "");
t.style.setProperty("--i", Math.min(i, 40));
fillTileText(t, decoded[i], showSp);
if (showId) { const d = document.createElement("span"); d.className = "tid"; d.textContent = ids[i]; t.append(d); }
frag.append(t);
}
box.append(frag);
if (ids.length > MAX_TILES) {
const p = document.createElement("p");
p.className = "empty"; p.style.width = "100%";
p.textContent = "Pokazano pierwsze " + nf.format(MAX_TILES) + " tokenów z " + nf.format(ids.length) + ".";
box.append(p);
}
$("legend").hidden = !hasFrag;
firstRender = false;
}
function showMessage(text, retry) {
const box = $("tokens");
box.replaceChildren();
const p = document.createElement("p");
p.className = retry ? "err" : "empty";
p.textContent = text;
if (retry) {
const b = document.createElement("button");
b.type = "button"; b.className = "btn"; b.textContent = "Spróbuj ponownie";
b.addEventListener("click", retry);
p.append(b);
}
box.append(p);
$("legend").hidden = true;
}
function setStats(tokens, text) {
const chars = countChars(text), words = countWords(text);
$("s-tok").textContent = tokens == null ? "-" : nf.format(tokens);
$("s-chr").textContent = nf.format(chars);
$("s-wrd").textContent = nf.format(words);
$("s-tpw").textContent = (tokens != null && words > 0) ? nf1.format(tokens / words) : "-";
}
/* ============================================================
Główny przebieg
============================================================ */
async function render() {
const my = ++runId;
const text = $("text").value;
const m = modelById(currentId);
if (!text) {
setStats(0, "");
showMessage("Wpisz tekst albo wybierz przykład, a tutaj pojawią się tokeny.");
updateBars(text);
return;
}
setStats(null, text);
let tok;
if (!loaded.has(currentId)) showMessage("Pobieram tokenizer " + m.label + "…");
try {
tok = await getTokenizer(currentId);
} catch (e) {
if (my !== runId) return;
console.error(e);
showMessage("Nie udało się pobrać tokenizera " + m.label + ". Sprawdź połączenie z internetem i spróbuj ponownie.", () => render());
updateBars(text);
return;
}
if (my !== runId) return;
let ids, decoded;
try {
ids = encodeIds(tok, text);
decoded = ids.slice(0, MAX_TILES).map((id) => tok.decode([id]));
// Tokenizery typu Llama dodają sztuczną spację na początku tekstu; nie pokazujemy jej, jeśli użytkownik jej nie wpisał.
if (decoded.length && decoded[0].startsWith(" ") && !text.startsWith(" ")) decoded[0] = decoded[0].slice(1);
} catch (e) {
console.error(e);
showMessage("Ten tokenizer nie poradził sobie z podanym tekstem.");
return;
}
setStats(ids.length, text);
drawTokens(ids, decoded);
updateBars(text);
}
/* ============================================================
Porównanie modeli
============================================================ */
function updateBars(text) {
const list = $("bars");
if (!compareOn) { list.replaceChildren(); $("insight").textContent = ""; return; }
const rows = MODELS.map((m, i) => {
const tok = loaded.get(m.id);
let n = null;
if (tok && text) { try { n = encodeIds(tok, text).length; } catch (_) { n = null; } }
return { m, i, n };
});
const max = Math.max(1, ...rows.map((r) => r.n ?? 0));
const words = countWords(text);
list.replaceChildren();
rows.forEach(({ m, i, n }) => {
const li = document.createElement("li");
li.className = "bar-row";
const b = document.createElement("button");
b.type = "button"; b.textContent = m.label;
if (m.id === currentId) b.setAttribute("aria-current", "true");
b.title = "Pokaż tokeny tego modelu";
b.addEventListener("click", () => setModel(m.id));
const track = document.createElement("div");
track.className = "bar-track";
if (n == null) {
const s = document.createElement("span");
s.className = "bar-na";
s.textContent = failed.has(m.id) ? "brak danych (nie udało się pobrać tokenizera)" : (text ? "ładowanie…" : "-");
track.append(s);
} else {
const bar = document.createElement("div");
bar.className = "bar c" + (i % 5);
bar.style.width = "calc(" + (n / max * 100).toFixed(1) + "% - 9rem)";
bar.style.minWidth = "6px";
const v = document.createElement("span");
v.className = "bar-val";
v.append(nf.format(n) + " " + plTokeny(n));
if (words > 0) { const sm = document.createElement("small"); sm.textContent = " (" + nf1.format(n / words) + " na słowo)"; v.append(sm); }
track.append(bar, v);
}
li.append(b, track);
list.append(li);
});
const have = rows.filter((r) => r.n != null);
if (have.length >= 2) {
const lo = have.reduce((a, b) => (b.n < a.n ? b : a));
const hi = have.reduce((a, b) => (b.n > a.n ? b : a));
$("insight").textContent = lo.n === hi.n
? "Wszystkie modele podzieliły ten tekst na tyle samo tokenów."
: "Najmniej tokenów: " + lo.m.label + " (" + nf.format(lo.n) + "). Najwięcej: " + hi.m.label + " (" + nf.format(hi.n) + "). Stosunek największej do najmniejszej liczby: " + nf2.format(hi.n / lo.n) + "×.";
} else {
$("insight").textContent = "";
}
}
async function runCompare() {
const btn = $("btn-cmp"), prog = $("cmp-progress");
compareOn = true;
btn.disabled = true;
const todo = MODELS.filter((m) => !loaded.has(m.id));
let done = 0;
updateBars($("text").value);
for (const m of todo) {
prog.textContent = "Pobieram tokenizer " + m.label + " (" + (done + 1) + " z " + todo.length + ")…";
try { await getTokenizer(m.id); } catch (e) { console.error(e); }
done++;
updateBars($("text").value);
}
prog.textContent = failed.size ? "Część tokenizerów nie została pobrana." : "";
btn.disabled = false;
btn.textContent = "Odśwież porównanie";
if (failed.size) btn.textContent = "Spróbuj ponownie pobrać brakujące";
}
/* ============================================================
Start
============================================================ */
buildChips();
buildExamples();
let timer = null;
$("text").addEventListener("input", () => { clearTimeout(timer); timer = setTimeout(render, DEBOUNCE_MS); });
$("opt-sp").addEventListener("change", render);
$("opt-id").addEventListener("change", render);
$("btn-cmp").addEventListener("click", runCompare);
$("text").value = params.get("text") ?? EXAMPLES[0].text;
render();
</script>
<noscript><div class="wrap"><p class="privacy">Do działania tokenizatora potrzebny jest JavaScript.</p></div></noscript>
</body>
</html>