Spaces:
Running
Running
Download index.html from HubertRNA/tokenizator: direct link, hf CLI and curl.
- Browser
- Download file 39.6 kB
-
https://huggingface.co/spaces/HubertRNA/tokenizator/resolve/main/index.html
- Command line
-
hf download hf://spaces/HubertRNA/tokenizator/index.html
-
curl -L -o index.html https://huggingface.co/spaces/HubertRNA/tokenizator/resolve/main/index.html
39.6 kB
| <html lang="pl"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Tokenizator - ReceptaNaAI.pl</title> | |
| <meta name="description" content="Zobacz, jak modele językowe dzielą tekst na tokeny. Działa w przeglądarce, tekst nigdzie nie jest wysyłany."> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=Montserrat:wght@600;700;800&family=Atkinson+Hyperlegible:wght@400;700&family=JetBrains+Mono:wght@500;700&display=swap" rel="stylesheet"> | |
| <style> | |
| :root { | |
| color-scheme: light; | |
| --navy: #12304F; | |
| --navy-2: #1B4F7A; | |
| --text: #1C2A3A; | |
| --muted: #4B5D70; | |
| --cream: #FBF4DD; | |
| --cream-hi: #FFF8DC; | |
| --gold-dark: #8A6420; | |
| --gold-mid: #C99A2E; | |
| --gold-light: #F3D98B; | |
| --gold: linear-gradient(135deg, #8A6420 0%, #D9AE45 22%, #FFF1B8 45%, #D4A93E 68%, #8A6420 100%); | |
| --coral-d: #B3361F; | |
| --font-head: "Montserrat", "Segoe UI", system-ui, sans-serif; | |
| --font-body: "Atkinson Hyperlegible", "Segoe UI", system-ui, sans-serif; | |
| --font-mono: "JetBrains Mono", ui-monospace, "Cascadia Code", Consolas, monospace; | |
| --gloss: linear-gradient(180deg, rgba(255,255,255,.55) 0, rgba(255,255,255,0) 55%); | |
| } | |
| * { box-sizing: border-box; } | |
| html { -webkit-text-size-adjust: 100%; } | |
| body { | |
| margin: 0; | |
| background: linear-gradient(180deg, #E9F3EE 0, #F4EBD3 100%); | |
| color: var(--text); | |
| font-family: var(--font-body); | |
| font-size: 1.0625rem; | |
| line-height: 1.55; | |
| } | |
| a { color: var(--navy-2); text-underline-offset: 3px; } | |
| a:hover { color: var(--coral-d); } | |
| :focus-visible { outline: 3px solid var(--navy); outline-offset: 3px; } | |
| .wrap { max-width: 1000px; margin: 0 auto; padding: 0 16px; } | |
| .sr { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; } | |
| /* ================= hero: niebo, miasto, kopuła ================= */ | |
| .hero { | |
| position: relative; overflow: hidden; text-align: center; padding: 30px 0 150px; | |
| background: | |
| radial-gradient(ellipse 190px 42px at 10% 18%, rgba(255,255,255,.85), transparent 70%), | |
| radial-gradient(ellipse 240px 50px at 88% 30%, rgba(255,255,255,.8), transparent 70%), | |
| radial-gradient(ellipse 150px 34px at 68% 9%, rgba(255,255,255,.7), transparent 70%), | |
| linear-gradient(180deg, #86C5E0 0%, #B9E0EC 48%, #E6F2EE 100%); | |
| border-bottom: 5px solid var(--gold-mid); | |
| } | |
| .hero > svg { position: absolute; pointer-events: none; } | |
| .hero .dome { inset: 0; width: 100%; height: 100%; } | |
| .hero .skyline { left: 0; bottom: 0; width: 100%; height: 190px; } | |
| .hero .plant { bottom: -14px; width: 230px; height: 210px; } | |
| .hero .plant-l { left: -26px; } | |
| .hero .plant-r { right: -26px; transform: scaleX(-1); } | |
| .hero-inner { position: relative; z-index: 1; } | |
| .brand { display: block; font-family: var(--font-head); font-weight: 700; color: var(--navy); text-decoration: none; letter-spacing: .02em; } | |
| .brand:hover { color: var(--coral-d); } | |
| .sign { | |
| position: relative; display: table; margin: 30px auto 0; | |
| padding: 14px clamp(22px, 5vw, 58px) 12px; | |
| border: 5px solid transparent; border-radius: 18px; | |
| background: | |
| radial-gradient(circle at 15px 15px, #A87A24 0 3px, transparent 3.8px) padding-box, | |
| radial-gradient(circle at calc(100% - 15px) 15px, #A87A24 0 3px, transparent 3.8px) padding-box, | |
| radial-gradient(circle at 15px calc(100% - 15px), #A87A24 0 3px, transparent 3.8px) padding-box, | |
| radial-gradient(circle at calc(100% - 15px) calc(100% - 15px), #A87A24 0 3px, transparent 3.8px) padding-box, | |
| linear-gradient(var(--cream), var(--cream)) padding-box, | |
| var(--gold) border-box; | |
| box-shadow: 0 10px 24px rgba(18,48,79,.28); | |
| } | |
| .sign::before { content: ""; position: absolute; inset: 6px; border: 1.5px solid var(--gold-mid); border-radius: 11px; pointer-events: none; } | |
| .sign .star { position: absolute; left: 50%; top: -21px; width: 28px; height: 28px; transform: translateX(-50%); } | |
| h1 { | |
| margin: 0; font-family: var(--font-head); font-weight: 800; text-transform: uppercase; | |
| font-size: clamp(1.9rem, 7.2vw, 3.7rem); letter-spacing: .03em; line-height: 1.15; color: var(--navy); | |
| } | |
| .hero-sub { margin: 22px 0 0; font-family: var(--font-head); font-weight: 700; font-size: clamp(1.05rem, 3vw, 1.35rem); color: var(--navy); } | |
| .lead { max-width: 58ch; margin: 8px auto 0; } | |
| .privacy { | |
| display: flex; align-items: center; gap: 14px; text-align: left; | |
| max-width: 700px; margin: 22px auto 0; padding: 12px 18px 12px 12px; | |
| border: 3px solid transparent; border-radius: 16px; | |
| background: linear-gradient(rgba(251,244,221,.94), rgba(251,244,221,.94)) padding-box, var(--gold) border-box; | |
| box-shadow: 0 6px 16px rgba(18,48,79,.18); | |
| } | |
| .privacy .lock { | |
| flex: none; width: 48px; height: 48px; border-radius: 50%; display: grid; place-items: center; | |
| border: 3px solid transparent; | |
| background: radial-gradient(circle at 35% 28%, #2D6A9E, #12304F 70%) padding-box, var(--gold) border-box; | |
| } | |
| .privacy p { margin: 0; } | |
| /* ================= panele ================= */ | |
| main { padding-bottom: 46px; } | |
| .panel { | |
| position: relative; margin-top: 54px; padding: 40px 22px 24px; border-radius: 22px; | |
| background: rgba(255,255,255,.6); | |
| -webkit-backdrop-filter: blur(8px); backdrop-filter: blur(8px); | |
| box-shadow: 0 12px 30px rgba(18,48,79,.13); | |
| } | |
| .panel::before { | |
| content: ""; position: absolute; inset: 0; border-radius: inherit; padding: 3px; background: var(--gold); pointer-events: none; | |
| -webkit-mask: linear-gradient(#000 0 0) content-box, linear-gradient(#000 0 0); | |
| -webkit-mask-composite: xor; | |
| mask: linear-gradient(#000 0 0) content-box, linear-gradient(#000 0 0); | |
| mask-composite: exclude; | |
| } | |
| .plaque { | |
| position: absolute; z-index: 2; top: 0; left: 50%; transform: translate(-50%, -50%); margin: 0; white-space: nowrap; | |
| padding: 7px 22px; border: 3px solid transparent; border-radius: 14px; | |
| background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box; | |
| font-family: var(--font-head); font-weight: 800; font-size: clamp(.86rem, 2.6vw, 1.08rem); | |
| letter-spacing: .04em; text-transform: uppercase; color: var(--navy); line-height: 1.2; | |
| box-shadow: 0 4px 10px rgba(18,48,79,.2); | |
| } | |
| .hint { color: var(--muted); margin: 0 0 14px; max-width: 64ch; } | |
| /* ================= kafelki tokenów ================= */ | |
| .tok { | |
| display: inline-block; position: relative; border: 2px solid var(--navy); border-radius: 8px; | |
| padding: 1px 7px 2px; font-family: var(--font-mono); font-weight: 500; font-size: 1.05rem; line-height: 1.45; | |
| white-space: pre-wrap; word-break: break-all; color: var(--navy); | |
| background-color: var(--fill, #B3E2E0); background-image: var(--gloss); | |
| box-shadow: 0 2px 0 rgba(18,48,79,.35); | |
| } | |
| .c0 { --fill: #B3E2E0; } .c1 { --fill: #F7B5A5; } .c2 { --fill: #F8DC8E; } .c3 { --fill: #C0CDEC; } .c4 { --fill: #C6E5BE; } | |
| .tok.frag { background-image: repeating-linear-gradient(45deg, rgba(18,48,79,.24) 0 3px, transparent 3px 7px), var(--gloss); } | |
| .sp, .nl { opacity: .45; } | |
| .tid { display: block; font-size: .72rem; font-weight: 700; line-height: 1.1; text-align: center; color: var(--navy-2); margin-top: 1px; } | |
| /* ================= wybór modelu ================= */ | |
| .chips { display: flex; flex-wrap: wrap; justify-content: center; gap: 12px; } | |
| .chip { position: relative; } | |
| .chip input { position: absolute; opacity: 0; inset: 0; width: 100%; height: 100%; margin: 0; cursor: pointer; } | |
| .chip span { | |
| display: inline-flex; align-items: center; gap: 8px; min-height: 46px; padding: 6px 22px; | |
| border: 3px solid transparent; border-radius: 999px; | |
| background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box; | |
| font-family: var(--font-head); font-weight: 700; font-size: .95rem; color: var(--navy); | |
| box-shadow: 0 3px 0 rgba(138,100,32,.55); transition: transform .08s, box-shadow .08s; | |
| } | |
| .chip input:hover + span { background: linear-gradient(var(--cream-hi), var(--cream-hi)) padding-box, var(--gold) border-box; } | |
| .chip input:checked + span { | |
| background: radial-gradient(circle at 35% 25%, #2B6395, #12304F 70%) padding-box, var(--gold) border-box; | |
| color: var(--gold-light); box-shadow: 0 1px 0 rgba(138,100,32,.55); transform: translateY(2px); | |
| } | |
| .chip input:focus-visible + span { outline: 3px solid var(--navy); outline-offset: 3px; } | |
| .chip small { font-family: var(--font-body); font-weight: 400; font-size: .8rem; } | |
| .model-note { min-height: 1.6em; margin: 14px 0 0; text-align: center; color: var(--muted); font-size: .95rem; } | |
| /* ================= wpisywanie ================= */ | |
| label.field { display: block; font-weight: 700; margin-bottom: 8px; } | |
| textarea { | |
| width: 100%; min-height: 130px; resize: vertical; padding: 12px 14px; | |
| font-family: var(--font-mono); font-size: 1.05rem; line-height: 1.5; color: var(--navy); | |
| border: 3px solid transparent; border-radius: 16px; | |
| background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box; | |
| box-shadow: inset 0 2px 8px rgba(18,48,79,.12); | |
| } | |
| textarea::placeholder { color: #6a7480; } | |
| .examples { margin-top: 16px; } | |
| .examples p { margin: 0 0 8px; font-weight: 700; } | |
| .ex-list { display: flex; flex-wrap: wrap; gap: 8px; } | |
| button { font: inherit; cursor: pointer; color: var(--navy); } | |
| .btn-ex { | |
| min-height: 40px; padding: 4px 16px; font-family: var(--font-head); font-weight: 700; font-size: .85rem; | |
| border: 2px solid transparent; border-radius: 999px; | |
| background: linear-gradient(var(--cream), var(--cream)) padding-box, var(--gold) border-box; | |
| } | |
| .btn-ex:hover { background: linear-gradient(var(--cream-hi), var(--cream-hi)) padding-box, var(--gold) border-box; } | |
| .btn { | |
| min-height: 46px; padding: 6px 22px; font-family: var(--font-head); font-weight: 800; font-size: .95rem; letter-spacing: .02em; | |
| border: 2px solid var(--gold-dark); border-radius: 999px; color: var(--navy); | |
| background: linear-gradient(180deg, #FFF1B8 0%, #E3B94E 55%, #C99A2E 100%); | |
| box-shadow: 0 3px 0 var(--gold-dark); | |
| } | |
| .btn:hover { background: linear-gradient(180deg, #FFF8D0 0%, #EBC55F 55%, #D6A93C 100%); } | |
| .btn:active { transform: translateY(2px); box-shadow: 0 1px 0 var(--gold-dark); } | |
| .btn[disabled] { opacity: .65; cursor: progress; } | |
| /* ================= medale ze statystykami ================= */ | |
| .stats { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; margin-bottom: 20px; } | |
| .stat { text-align: center; } | |
| .coin { | |
| display: grid; place-items: center; width: 112px; height: 112px; margin: 0 auto 8px; border-radius: 50%; | |
| border: 5px solid transparent; | |
| background: radial-gradient(circle at 35% 28%, #2D6A9E 0%, #164268 55%, #0C2742 100%) padding-box, var(--gold) border-box; | |
| box-shadow: 0 6px 14px rgba(18,48,79,.3), inset 0 0 0 2px #0C2742, inset 0 0 0 3.5px rgba(243,217,139,.7); | |
| } | |
| .coin .v { font-family: var(--font-head); font-weight: 800; font-size: 1.85rem; line-height: 1; color: var(--gold-light); white-space: nowrap; } | |
| .stat .k { display: block; font-size: .92rem; color: var(--muted); } | |
| /* ================= wynik ================= */ | |
| .out-tools { display: flex; flex-wrap: wrap; justify-content: center; gap: 8px 24px; margin: 0 0 14px; } | |
| .toggle { display: inline-flex; align-items: center; gap: 8px; font-weight: 700; cursor: pointer; min-height: 32px; } | |
| .toggle input { width: 20px; height: 20px; accent-color: var(--navy); margin: 0; } | |
| #tokens { | |
| display: flex; flex-wrap: wrap; gap: 8px 6px; align-items: flex-start; min-height: 120px; padding: 16px; | |
| background: #FBF7E8; border: 2px solid rgba(201,154,46,.75); border-radius: 16px; | |
| box-shadow: inset 0 2px 8px rgba(18,48,79,.12); | |
| } | |
| #tokens .empty { color: var(--muted); margin: 0; align-self: center; } | |
| #tokens .err { margin: 0; width: 100%; } | |
| #tokens .err .btn { margin-top: 12px; display: block; } | |
| .legend { margin: 12px 0 0; color: var(--muted); font-size: .95rem; max-width: 72ch; } | |
| .legend .tok { font-size: .85rem; padding: 0 6px; box-shadow: none; vertical-align: 1px; } | |
| @media (prefers-reduced-motion: no-preference) { | |
| .first .tok { animation: pop .35s cubic-bezier(.3,1.6,.5,1) both; animation-delay: calc(var(--i, 0) * 28ms); } | |
| @keyframes pop { from { transform: scale(.5) rotate(-4deg); opacity: 0; } to { transform: none; opacity: 1; } } | |
| } | |
| /* ================= porównanie ================= */ | |
| .compare-head { display: flex; flex-wrap: wrap; align-items: center; gap: 12px 18px; } | |
| .progress { color: var(--muted); font-size: .95rem; } | |
| #bars { list-style: none; margin: 18px 0 0; padding: 0; display: grid; gap: 10px; } | |
| .bar-row { display: grid; grid-template-columns: 8.5rem 1fr; gap: 12px; align-items: center; } | |
| .bar-row button { text-align: left; background: none; border: 0; padding: 4px 0; min-height: 40px; font-family: var(--font-head); font-weight: 700; font-size: .95rem; } | |
| .bar-row button[aria-current="true"] { color: var(--coral-d); } | |
| .bar-row button[aria-current="true"]::after { content: " (wybrany)"; font-family: var(--font-body); font-weight: 400; font-size: .8rem; color: var(--muted); } | |
| .bar-track { display: flex; align-items: center; gap: 10px; } | |
| .bar { | |
| height: 32px; min-width: 6px; border: 2px solid var(--navy); border-radius: 999px; | |
| background-color: var(--fill); background-image: var(--gloss); box-shadow: 0 2px 0 rgba(18,48,79,.35); transition: width .25s; | |
| } | |
| .bar-val { font-family: var(--font-mono); font-weight: 700; white-space: nowrap; color: var(--navy); } | |
| .bar-val small { font-family: var(--font-body); font-weight: 400; color: var(--muted); } | |
| .bar-na { color: var(--muted); } | |
| .insight { margin: 16px 0 0; font-family: var(--font-head); font-weight: 700; font-size: .95rem; color: var(--navy); max-width: 70ch; } | |
| /* ================= uwagi ================= */ | |
| .notes { display: grid; grid-template-columns: repeat(2, 1fr); gap: 0 34px; padding: 0; margin: 0; list-style: none; } | |
| .notes li { padding: 14px 0; border-top: 2px solid rgba(201,154,46,.7); } | |
| .notes b { font-family: var(--font-head); font-weight: 700; color: var(--navy); } | |
| /* ================= stopka ================= */ | |
| footer { background: var(--navy); color: #DCE6F0; border-top: 5px solid var(--gold-mid); padding: 26px 0 36px; font-size: .92rem; } | |
| footer p { margin: 0 0 8px; max-width: 90ch; } | |
| footer strong { color: var(--gold-light); } | |
| footer a { color: var(--gold-light); } | |
| footer a:hover { color: #fff; } | |
| @media (max-width: 900px) { .hero .plant { display: none; } } | |
| @media (max-width: 720px) { | |
| .hero { padding-bottom: 120px; } | |
| .panel { padding: 38px 14px 20px; } | |
| .stats { grid-template-columns: repeat(2, 1fr); row-gap: 14px; } | |
| .coin { width: 96px; height: 96px; } | |
| .coin .v { font-size: 1.55rem; } | |
| .notes { grid-template-columns: 1fr; } | |
| .bar-row { grid-template-columns: 1fr; gap: 0; } | |
| .bar-row button { min-height: 36px; } | |
| .privacy { text-align: left; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <header class="hero"> | |
| <svg class="dome" viewBox="0 0 1000 100" preserveAspectRatio="none" aria-hidden="true"> | |
| <path d="M 4 100 V 26 A 496 22 0 0 1 996 26 V 100" fill="none" stroke="#C99A2E" stroke-width="5" vector-effect="non-scaling-stroke" opacity=".85"/> | |
| <path d="M 14 100 V 30 A 486 21 0 0 1 986 30 V 100" fill="none" stroke="#FFF1B8" stroke-width="1.5" vector-effect="non-scaling-stroke" opacity=".9"/> | |
| </svg> | |
| <svg class="skyline" viewBox="0 0 1000 190" preserveAspectRatio="xMidYMax slice" aria-hidden="true"> | |
| <g fill="#D3EAEE" opacity=".9"> | |
| <rect x="60" y="110" width="40" height="80"/><rect x="150" y="90" width="34" height="100"/><rect x="330" y="120" width="46" height="70"/> | |
| <rect x="560" y="105" width="44" height="85"/><rect x="760" y="95" width="38" height="95"/><rect x="900" y="115" width="50" height="75"/> | |
| <path d="M200 190 A32 32 0 0 1 264 190 Z"/><path d="M820 190 A30 30 0 0 1 880 190 Z"/> | |
| </g> | |
| <g fill="#B5DCE4"> | |
| <rect x="296" y="46" width="6" height="144"/><ellipse cx="299" cy="56" rx="28" ry="7"/><rect x="297" y="26" width="4" height="22"/> | |
| <path d="M690 190 L699 30 L708 190 Z"/><ellipse cx="699" cy="62" rx="20" ry="5"/> | |
| <rect x="110" y="128" width="30" height="62"/><rect x="420" y="100" width="36" height="90"/><rect x="480" y="136" width="52" height="54"/> | |
| <rect x="610" y="124" width="34" height="66"/><rect x="940" y="96" width="30" height="94"/> | |
| <path d="M380 190 A44 44 0 0 1 468 190 Z" fill="#A4D3DD"/> | |
| </g> | |
| <g fill="#F3D98B" opacity=".9"><rect x="420" y="112" width="4" height="6"/><rect x="430" y="124" width="4" height="6"/><rect x="940" y="110" width="4" height="6"/><rect x="954" y="128" width="4" height="6"/><rect x="560" y="120" width="4" height="6"/></g> | |
| </svg> | |
| <svg class="plant plant-l" viewBox="0 0 230 210" aria-hidden="true"> | |
| <g stroke-linejoin="round"> | |
| <g transform="translate(78 210) rotate(-64) scale(.8)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(46) scale(.75)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(-34)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#2F7D55" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(22) scale(.95)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(-6) scale(1.15)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g> | |
| </g> | |
| </svg> | |
| <svg class="plant plant-r" viewBox="0 0 230 210" aria-hidden="true"> | |
| <g stroke-linejoin="round"> | |
| <g transform="translate(78 210) rotate(-64) scale(.8)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(46) scale(.75)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(-34)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#2F7D55" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(22) scale(.95)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#256B47" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#8BD0A0" stroke-width="1.5"/></g> | |
| <g transform="translate(78 210) rotate(-6) scale(1.15)"><path d="M0 0 C-20 -35 -18 -85 0 -125 C18 -85 20 -35 0 0Z" fill="#3E9467" stroke="#1B5A3B" stroke-width="1.5"/><path d="M0 -4 L0 -112" stroke="#B5E6C4" stroke-width="1.5"/></g> | |
| </g> | |
| </svg> | |
| <div class="wrap hero-inner"> | |
| <a class="brand" href="https://receptanaai.pl" target="_blank" rel="noopener">ReceptaNaAI.pl</a> | |
| <div class="sign"> | |
| <svg class="star" viewBox="0 0 24 24" aria-hidden="true"><defs><linearGradient id="gs" x1="0" y1="0" x2="1" y2="1"><stop offset="0" stop-color="#FFF1B8"/><stop offset="1" stop-color="#C99A2E"/></linearGradient></defs><path d="M12 0 C12.8 7 17 11.2 24 12 C17 12.8 12.8 17 12 24 C11.2 17 7 12.8 0 12 C7 11.2 11.2 7 12 0Z" fill="url(#gs)" stroke="#8A6420" stroke-width=".8"/></svg> | |
| <h1>Tokenizator</h1> | |
| </div> | |
| <p class="hero-sub">Jak model językowy czyta Twój tekst?</p> | |
| <p class="lead">Model nie widzi liter ani słów. Widzi tokeny: kawałki tekstu, na które tnie go tokenizer. Wpisz zdanie i zobacz, jak potnie je wybrany model.</p> | |
| <div class="privacy"> | |
| <span class="lock" aria-hidden="true"><svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#F3D98B" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="5" y="11" width="14" height="10" rx="2" fill="#F3D98B" fill-opacity=".25"/><path d="M8 11V8a4 4 0 0 1 8 0v3"/></svg></span> | |
| <p><strong>Tekst jest przetwarzany w Twojej przeglądarce i nigdzie nie jest wysyłany.</strong> Mimo to nie wpisuj tu danych pacjentów ani innych danych wrażliwych.</p> | |
| </div> | |
| </div> | |
| </header> | |
| <main class="wrap"> | |
| <section class="panel" aria-labelledby="lg-model"> | |
| <h2 class="plaque" id="lg-model">1. Wybierz model</h2> | |
| <div class="chips" id="chips" role="radiogroup" aria-labelledby="lg-model"></div> | |
| <p class="model-note" id="model-note" aria-live="polite"></p> | |
| </section> | |
| <section class="panel" aria-labelledby="h-text"> | |
| <h2 class="plaque" id="h-text">2. Wpisz tekst</h2> | |
| <label class="field" for="text">Tekst do podzielenia na tokeny</label> | |
| <textarea id="text" spellcheck="false" placeholder="Wpisz zdanie albo wybierz przykład poniżej"></textarea> | |
| <div class="examples"> | |
| <p>Przykłady do porównania</p> | |
| <div class="ex-list" id="examples"></div> | |
| </div> | |
| </section> | |
| <section class="panel" aria-labelledby="h-out"> | |
| <h2 class="plaque" id="h-out">3. Zobacz tokeny</h2> | |
| <div class="stats" id="stats" aria-live="polite"> | |
| <div class="stat"><span class="coin"><span class="v" id="s-tok">0</span></span><span class="k">tokenów</span></div> | |
| <div class="stat"><span class="coin"><span class="v" id="s-chr">0</span></span><span class="k">znaków</span></div> | |
| <div class="stat"><span class="coin"><span class="v" id="s-wrd">0</span></span><span class="k">słów</span></div> | |
| <div class="stat"><span class="coin"><span class="v" id="s-tpw">-</span></span><span class="k">tokenów na słowo</span></div> | |
| </div> | |
| <div class="out-tools"> | |
| <label class="toggle"><input type="checkbox" id="opt-sp" checked> Pokaż spacje</label> | |
| <label class="toggle"><input type="checkbox" id="opt-id"> Pokaż numery tokenów</label> | |
| </div> | |
| <div id="tokens" role="region" aria-label="Tokeny" aria-live="polite"></div> | |
| <p class="legend" id="legend" hidden><span class="tok frag" aria-hidden="true">�</span> Kratka oznacza fragment znaku zapisanego w kilku bajtach. Tak bywają cięte rzadsze litery, np. polskie znaki diakrytyczne.</p> | |
| </section> | |
| <section class="panel" aria-labelledby="h-cmp"> | |
| <h2 class="plaque" id="h-cmp">4. Porównaj modele</h2> | |
| <p class="hint">Ten sam tekst, różne tokenizery. Porównanie pobiera pliki wszystkich tokenizerów, więc za pierwszym razem może chwilę potrwać.</p> | |
| <div class="compare-head"> | |
| <button class="btn" id="btn-cmp" type="button">Porównaj wszystkie modele</button> | |
| <span class="progress" id="cmp-progress" aria-live="polite"></span> | |
| </div> | |
| <ul id="bars" aria-label="Liczba tokenów w poszczególnych modelach"></ul> | |
| <p class="insight" id="insight" aria-live="polite"></p> | |
| </section> | |
| <section class="panel" aria-labelledby="h-notes"> | |
| <h2 class="plaque" id="h-notes">Co warto zauważyć</h2> | |
| <ul class="notes"> | |
| <li><b>Token to nie słowo.</b> Krótkie, częste słowa zwykle zajmują jeden token, a długie lub rzadkie dzielą się na kilka kawałków. Sprawdź to na nazwach leków.</li> | |
| <li><b>Ten sam sens, inna liczba tokenów.</b> Porównaj zdanie po polsku z tym samym zdaniem po angielsku i zobacz, czy któryś model potrzebuje więcej tokenów.</li> | |
| <li><b>Tokeny to waluta modeli.</b> Okno kontekstu i koszt zapytań w API liczy się w tokenach, a nie w znakach.</li> | |
| <li><b>Każdy model ma własny tokenizer.</b> Liczba tokenów tego samego tekstu zależy od modelu, więc używaj porównania.</li> | |
| </ul> | |
| </section> | |
| </main> | |
| <footer> | |
| <div class="wrap"> | |
| <p><strong>Skąd to się bierze.</strong> Tokenizacja działa lokalnie dzięki bibliotece Transformers.js (Hugging Face, licencja Apache 2.0). Przeglądarka pobiera bibliotekę z serwisu jsDelivr, pliki tokenizerów z Hugging Face Hub i czcionki z Google Fonts. Twój tekst nie jest do nich wysyłany.</p> | |
| <p><strong>Licencje.</strong> Strona powstała z inspiracji projektem „The Tokenizer Playground” (Xenova, Hugging Face, Apache 2.0). Tokenizery pochodzą z repozytoriów społeczności Xenova i podlegają licencjom modeli, z których wywodzą się. Tokenizer oznaczony jako Claude jest nieoficjalny.</p> | |
| <p><strong>Uwaga.</strong> Liczby tokenów służą celom edukacyjnym. Rzeczywiste zużycie tokenów w usługach komercyjnych może się różnić, np. przez znaczniki systemowe. Tu nie liczymy znaczników specjalnych.</p> | |
| <p>© Recepta Na AI sp. z o.o. · <a href="https://receptanaai.pl" target="_blank" rel="noopener">ReceptaNaAI.pl</a></p> | |
| </div> | |
| </footer> | |
| <script type="module"> | |
| /* ============================================================ | |
| Konfiguracja: modele (tokenizery z Hugging Face Hub) | |
| Aby dodać model, dopisz wpis do tablicy MODELS. | |
| ============================================================ */ | |
| const LIB_URL = "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.1"; | |
| const MODELS = [ | |
| { id: "Xenova/gpt-4", label: "GPT-4", note: "Tokenizer rodziny GPT-4 i GPT-3.5. Nowsze modele OpenAI mogą używać innych tokenizerów." }, | |
| { id: "Xenova/llama-3-tokenizer", label: "Llama 3", note: "Tokenizer rodziny Llama 3 (Meta)." }, | |
| { id: "Xenova/mistral-tokenizer-v3", label: "Mistral v3", note: "Tokenizer Mistral v3." }, | |
| { id: "Xenova/gemma-tokenizer", label: "Gemma", note: "Tokenizer rodziny Gemma (Google)." }, | |
| { id: "Xenova/claude-tokenizer", label: "Claude", badge: "nieoficjalny", note: "Tokenizer społecznościowy (Xenova), nieoficjalny. Liczby są orientacyjne i mogą różnić się od tych w faktycznych modelach Claude." }, | |
| ]; | |
| const EXAMPLES = [ | |
| { label: "Zdanie po polsku", text: "Pacjent z kardiomiopatią przerostową otrzymał bisoprolol i amiodaron." }, | |
| { label: "To samo po angielsku", text: "The patient with hypertrophic cardiomyopathy received bisoprolol and amiodarone." }, | |
| { label: "Skróty z kardiologii", text: "EKG: SR 72/min, LVEF 55%, NT-proBNP 340 pg/ml, hs-TnT 12 ng/l." }, | |
| { label: "Nazwy leków", text: "metoprolol, rywaroksaban, atorwastatyna, ezetymib, sakubitryl/walsartan" }, | |
| { label: "Łacina kliniczna", text: "Fibrillatio atriorum paroxysmalis, status post ablationem." }, | |
| { label: "Zdanie bazowe", text: "Ala ma kota, a kot ma Alę." }, | |
| ]; | |
| const MAX_TILES = 1500; // tyle kafelków maksymalnie rysujemy | |
| const DEBOUNCE_MS = 150; | |
| /* ============================================================ | |
| Pomocnicze | |
| ============================================================ */ | |
| const $ = (id) => document.getElementById(id); | |
| const nf = new Intl.NumberFormat("pl-PL"); | |
| const nf1 = new Intl.NumberFormat("pl-PL", { minimumFractionDigits: 1, maximumFractionDigits: 1 }); | |
| const nf2 = new Intl.NumberFormat("pl-PL", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); | |
| function plTokeny(n) { | |
| if (n === 1) return "token"; | |
| const m10 = n % 10, m100 = n % 100; | |
| return (m10 >= 2 && m10 <= 4 && !(m100 >= 12 && m100 <= 14)) ? "tokeny" : "tokenów"; | |
| } | |
| const countChars = (s) => Array.from(s).length; | |
| const countWords = (s) => s.trim() ? s.trim().split(/\s+/).length : 0; | |
| const modelById = (id) => MODELS.find((m) => m.id === id); | |
| /* ============================================================ | |
| Stan | |
| ============================================================ */ | |
| const params = new URLSearchParams(location.search); | |
| let currentId = modelById(params.get("model")) ? params.get("model") : MODELS[0].id; | |
| let runId = 0; | |
| let firstRender = true; | |
| let compareOn = false; | |
| const loaded = new Map(); // id -> tokenizer (gotowy) | |
| const loading = new Map(); // id -> Promise<tokenizer> | |
| const failed = new Set(); // id, którym nie udało się załadować | |
| let lib = null; | |
| /* ============================================================ | |
| Ładowanie biblioteki i tokenizerów | |
| ============================================================ */ | |
| async function getLib() { | |
| if (lib) return lib; | |
| lib = await import(LIB_URL); | |
| lib.env.allowLocalModels = false; | |
| return lib; | |
| } | |
| function fixDecoder(tok) { | |
| // Tokenizery typu Llama/Mistral kończą dekoder krokiem StripDecoder, który obcina wiodącą spację tokenu. | |
| // Zdejmujemy go, żeby przy pojedynczych tokenach było widać, do którego tokenu należy spacja. | |
| try { | |
| const list = tok.decoder?.decoders; | |
| if (Array.isArray(list) && list.at(-1)?.constructor?.name === "StripDecoder") list.pop(); | |
| } catch (_) { /* wygląd nie jest krytyczny */ } | |
| return tok; | |
| } | |
| function getTokenizer(id) { | |
| if (loaded.has(id)) return Promise.resolve(loaded.get(id)); | |
| if (loading.has(id)) return loading.get(id); | |
| const p = (async () => { | |
| const { AutoTokenizer } = await getLib(); | |
| const tok = fixDecoder(await AutoTokenizer.from_pretrained(id)); | |
| loaded.set(id, tok); | |
| failed.delete(id); | |
| return tok; | |
| })(); | |
| loading.set(id, p); | |
| p.catch(() => { failed.add(id); }).finally(() => loading.delete(id)); | |
| return p; | |
| } | |
| function encodeIds(tok, text) { | |
| // Bez znaczników specjalnych (np. BOS), żeby liczby były porównywalne między modelami. | |
| const ids = tok.encode(text, { add_special_tokens: false }); | |
| return Array.from(ids, Number); | |
| } | |
| /* ============================================================ | |
| Interfejs: wybór modelu | |
| ============================================================ */ | |
| function buildChips() { | |
| const box = $("chips"); | |
| MODELS.forEach((m) => { | |
| const lab = document.createElement("label"); | |
| lab.className = "chip"; | |
| const inp = document.createElement("input"); | |
| inp.type = "radio"; inp.name = "model"; inp.value = m.id; inp.checked = m.id === currentId; | |
| inp.addEventListener("change", () => setModel(m.id)); | |
| const span = document.createElement("span"); | |
| span.append(m.label); | |
| if (m.badge) { const s = document.createElement("small"); s.textContent = "(" + m.badge + ")"; span.append(s); } | |
| lab.append(inp, span); | |
| box.append(lab); | |
| }); | |
| updateNote(); | |
| } | |
| function updateNote() { | |
| const m = modelById(currentId); | |
| $("model-note").textContent = m ? m.note : ""; | |
| } | |
| function setModel(id) { | |
| currentId = id; | |
| document.querySelectorAll('input[name="model"]').forEach((r) => { r.checked = r.value === id; }); | |
| updateNote(); | |
| render(); | |
| } | |
| function buildExamples() { | |
| const box = $("examples"); | |
| EXAMPLES.forEach((ex) => { | |
| const b = document.createElement("button"); | |
| b.type = "button"; b.className = "btn-ex"; b.textContent = ex.label; | |
| b.addEventListener("click", () => { $("text").value = ex.text; render(); }); | |
| box.append(b); | |
| }); | |
| } | |
| /* ============================================================ | |
| Rysowanie kafelków (tekst użytkownika wstawiamy wyłącznie przez textContent) | |
| ============================================================ */ | |
| function fillTileText(el, text, showSpaces) { | |
| if (!showSpaces) { el.textContent = text; return; } | |
| let buf = ""; | |
| const flush = () => { if (buf) { el.append(document.createTextNode(buf)); buf = ""; } }; | |
| for (const ch of text) { | |
| if (ch === " ") { flush(); const s = document.createElement("span"); s.className = "sp"; s.textContent = "·"; el.append(s); } | |
| else if (ch === "\n") { flush(); const s = document.createElement("span"); s.className = "nl"; s.textContent = "↵"; el.append(s); el.append(document.createTextNode("\n")); } | |
| else if (ch === "\t") { flush(); const s = document.createElement("span"); s.className = "sp"; s.textContent = "⇥"; el.append(s); } | |
| else buf += ch; | |
| } | |
| flush(); | |
| } | |
| function drawTokens(ids, decoded) { | |
| const box = $("tokens"); | |
| box.replaceChildren(); | |
| box.classList.toggle("first", firstRender); | |
| const showSp = $("opt-sp").checked, showId = $("opt-id").checked; | |
| const n = Math.min(ids.length, MAX_TILES); | |
| let hasFrag = false; | |
| const frag = document.createDocumentFragment(); | |
| for (let i = 0; i < n; i++) { | |
| const t = document.createElement("span"); | |
| const bad = decoded[i].includes("�"); | |
| if (bad) hasFrag = true; | |
| t.className = "tok c" + (i % 5) + (bad ? " frag" : ""); | |
| t.style.setProperty("--i", Math.min(i, 40)); | |
| fillTileText(t, decoded[i], showSp); | |
| if (showId) { const d = document.createElement("span"); d.className = "tid"; d.textContent = ids[i]; t.append(d); } | |
| frag.append(t); | |
| } | |
| box.append(frag); | |
| if (ids.length > MAX_TILES) { | |
| const p = document.createElement("p"); | |
| p.className = "empty"; p.style.width = "100%"; | |
| p.textContent = "Pokazano pierwsze " + nf.format(MAX_TILES) + " tokenów z " + nf.format(ids.length) + "."; | |
| box.append(p); | |
| } | |
| $("legend").hidden = !hasFrag; | |
| firstRender = false; | |
| } | |
| function showMessage(text, retry) { | |
| const box = $("tokens"); | |
| box.replaceChildren(); | |
| const p = document.createElement("p"); | |
| p.className = retry ? "err" : "empty"; | |
| p.textContent = text; | |
| if (retry) { | |
| const b = document.createElement("button"); | |
| b.type = "button"; b.className = "btn"; b.textContent = "Spróbuj ponownie"; | |
| b.addEventListener("click", retry); | |
| p.append(b); | |
| } | |
| box.append(p); | |
| $("legend").hidden = true; | |
| } | |
| function setStats(tokens, text) { | |
| const chars = countChars(text), words = countWords(text); | |
| $("s-tok").textContent = tokens == null ? "-" : nf.format(tokens); | |
| $("s-chr").textContent = nf.format(chars); | |
| $("s-wrd").textContent = nf.format(words); | |
| $("s-tpw").textContent = (tokens != null && words > 0) ? nf1.format(tokens / words) : "-"; | |
| } | |
| /* ============================================================ | |
| Główny przebieg | |
| ============================================================ */ | |
| async function render() { | |
| const my = ++runId; | |
| const text = $("text").value; | |
| const m = modelById(currentId); | |
| if (!text) { | |
| setStats(0, ""); | |
| showMessage("Wpisz tekst albo wybierz przykład, a tutaj pojawią się tokeny."); | |
| updateBars(text); | |
| return; | |
| } | |
| setStats(null, text); | |
| let tok; | |
| if (!loaded.has(currentId)) showMessage("Pobieram tokenizer " + m.label + "…"); | |
| try { | |
| tok = await getTokenizer(currentId); | |
| } catch (e) { | |
| if (my !== runId) return; | |
| console.error(e); | |
| showMessage("Nie udało się pobrać tokenizera " + m.label + ". Sprawdź połączenie z internetem i spróbuj ponownie.", () => render()); | |
| updateBars(text); | |
| return; | |
| } | |
| if (my !== runId) return; | |
| let ids, decoded; | |
| try { | |
| ids = encodeIds(tok, text); | |
| decoded = ids.slice(0, MAX_TILES).map((id) => tok.decode([id])); | |
| // Tokenizery typu Llama dodają sztuczną spację na początku tekstu; nie pokazujemy jej, jeśli użytkownik jej nie wpisał. | |
| if (decoded.length && decoded[0].startsWith(" ") && !text.startsWith(" ")) decoded[0] = decoded[0].slice(1); | |
| } catch (e) { | |
| console.error(e); | |
| showMessage("Ten tokenizer nie poradził sobie z podanym tekstem."); | |
| return; | |
| } | |
| setStats(ids.length, text); | |
| drawTokens(ids, decoded); | |
| updateBars(text); | |
| } | |
| /* ============================================================ | |
| Porównanie modeli | |
| ============================================================ */ | |
| function updateBars(text) { | |
| const list = $("bars"); | |
| if (!compareOn) { list.replaceChildren(); $("insight").textContent = ""; return; } | |
| const rows = MODELS.map((m, i) => { | |
| const tok = loaded.get(m.id); | |
| let n = null; | |
| if (tok && text) { try { n = encodeIds(tok, text).length; } catch (_) { n = null; } } | |
| return { m, i, n }; | |
| }); | |
| const max = Math.max(1, ...rows.map((r) => r.n ?? 0)); | |
| const words = countWords(text); | |
| list.replaceChildren(); | |
| rows.forEach(({ m, i, n }) => { | |
| const li = document.createElement("li"); | |
| li.className = "bar-row"; | |
| const b = document.createElement("button"); | |
| b.type = "button"; b.textContent = m.label; | |
| if (m.id === currentId) b.setAttribute("aria-current", "true"); | |
| b.title = "Pokaż tokeny tego modelu"; | |
| b.addEventListener("click", () => setModel(m.id)); | |
| const track = document.createElement("div"); | |
| track.className = "bar-track"; | |
| if (n == null) { | |
| const s = document.createElement("span"); | |
| s.className = "bar-na"; | |
| s.textContent = failed.has(m.id) ? "brak danych (nie udało się pobrać tokenizera)" : (text ? "ładowanie…" : "-"); | |
| track.append(s); | |
| } else { | |
| const bar = document.createElement("div"); | |
| bar.className = "bar c" + (i % 5); | |
| bar.style.width = "calc(" + (n / max * 100).toFixed(1) + "% - 9rem)"; | |
| bar.style.minWidth = "6px"; | |
| const v = document.createElement("span"); | |
| v.className = "bar-val"; | |
| v.append(nf.format(n) + " " + plTokeny(n)); | |
| if (words > 0) { const sm = document.createElement("small"); sm.textContent = " (" + nf1.format(n / words) + " na słowo)"; v.append(sm); } | |
| track.append(bar, v); | |
| } | |
| li.append(b, track); | |
| list.append(li); | |
| }); | |
| const have = rows.filter((r) => r.n != null); | |
| if (have.length >= 2) { | |
| const lo = have.reduce((a, b) => (b.n < a.n ? b : a)); | |
| const hi = have.reduce((a, b) => (b.n > a.n ? b : a)); | |
| $("insight").textContent = lo.n === hi.n | |
| ? "Wszystkie modele podzieliły ten tekst na tyle samo tokenów." | |
| : "Najmniej tokenów: " + lo.m.label + " (" + nf.format(lo.n) + "). Najwięcej: " + hi.m.label + " (" + nf.format(hi.n) + "). Stosunek największej do najmniejszej liczby: " + nf2.format(hi.n / lo.n) + "×."; | |
| } else { | |
| $("insight").textContent = ""; | |
| } | |
| } | |
| async function runCompare() { | |
| const btn = $("btn-cmp"), prog = $("cmp-progress"); | |
| compareOn = true; | |
| btn.disabled = true; | |
| const todo = MODELS.filter((m) => !loaded.has(m.id)); | |
| let done = 0; | |
| updateBars($("text").value); | |
| for (const m of todo) { | |
| prog.textContent = "Pobieram tokenizer " + m.label + " (" + (done + 1) + " z " + todo.length + ")…"; | |
| try { await getTokenizer(m.id); } catch (e) { console.error(e); } | |
| done++; | |
| updateBars($("text").value); | |
| } | |
| prog.textContent = failed.size ? "Część tokenizerów nie została pobrana." : ""; | |
| btn.disabled = false; | |
| btn.textContent = "Odśwież porównanie"; | |
| if (failed.size) btn.textContent = "Spróbuj ponownie pobrać brakujące"; | |
| } | |
| /* ============================================================ | |
| Start | |
| ============================================================ */ | |
| buildChips(); | |
| buildExamples(); | |
| let timer = null; | |
| $("text").addEventListener("input", () => { clearTimeout(timer); timer = setTimeout(render, DEBOUNCE_MS); }); | |
| $("opt-sp").addEventListener("change", render); | |
| $("opt-id").addEventListener("change", render); | |
| $("btn-cmp").addEventListener("click", runCompare); | |
| $("text").value = params.get("text") ?? EXAMPLES[0].text; | |
| render(); | |
| </script> | |
| <noscript><div class="wrap"><p class="privacy">Do działania tokenizatora potrzebny jest JavaScript.</p></div></noscript> | |
| </body> | |
| </html> | |