"""The demo's page and its CPU views, built without a model. Skipped when the `[app]` extra (gradio) is not installed. The views themselves are plain HTML and need only numpy, so most of what is checked here is the markup they produce. """ import html import importlib import json import os import re import numpy as np import pytest from test_runtime import make_trace from ulens import runtime gr = pytest.importorskip("gradio") @pytest.fixture(scope="module") def app(): """The module with the import-time model load switched off.""" os.environ["ULENS_APP_LOAD_MODEL"] = "0" from ulens import app return importlib.reload(app) # honour the flag even if imported earlier in the session def _payload(series_html: str) -> dict: m = re.search(r'', series_html, re.DOTALL) assert m, "the series view must carry the per-token data the hover reads" return json.loads(html.unescape(m.group(1))) def test_build_returns_blocks_without_loading_a_model(app): assert app.RUNNER is None demo = app.build() assert isinstance(demo, gr.Blocks) def test_the_theme_survives_gradio_s_built_in_comparison(app): """`launch` (with analytics on) compares the theme to every built-in theme, font by font; string fonts raise there. The comparison must run and the theme must not be a built-in.""" from gradio import utils # where launch() takes BUILT_IN_THEMES from t = app.theme() assert not any(t.to_dict() == b.to_dict() for b in utils.BUILT_IN_THEMES.values()) assert "system-ui" in t.font def test_the_page_has_one_palette_in_both_theme_modes(app): """Gradio's dark mode recolours every descendant of an HTML block with its dark text colour, on top of this page's light surface. The theme therefore carries no dark values at all: the `.dark` block Gradio emits must equal the root block, variable for variable.""" t = app.theme() darks = [a for a in vars(t) if a.endswith("_dark")] assert darks and all(getattr(t, a) == getattr(t, a[:-5]) for a in darks) t._get_computed_value("body_background_fill_dark") # what launch() reads; must not raise css = t._get_theme_css() root, dark = css.split(":root.dark", 1) pairs = re.compile(r"--([\w-]+): (.*?);") light_vars = dict(pairs.findall(root[root.index(":root {"):])) dark_vars = dict(pairs.findall(dark)) assert light_vars["body-text-color"] == app.INK assert light_vars["body-background-fill"] == app.SURFACE for name, value in light_vars.items(): assert dark_vars[name] == value, name assert "color-scheme: light" in app.CSS def test_every_view_renders_from_a_synthetic_trace(app): tr = make_trace() head, tmap, ser = app.views(tr, smooth=9) assert head == "" # no status line: the controls and labels say it all # the map: one span per token, three segments, the boundary marked assert tmap.count('data-i="') == tr.n_tokens assert all(f'data-i="{i}"' in tmap for i in range(tr.n_tokens)) for seg in ("ul-seg-prompt", "ul-seg-think", "ul-seg-answer"): assert seg in tmap assert f"end of think · token {tr.eot}" in tmap # the series: one directly labelled row per axis, then the entropy, each with a raw underlay assert ser.count('class="ul-row') == tr.n_axes + 1 assert ser.count('class="smooth"') == tr.n_axes + 1 assert ser.count('class="raw"') == tr.n_axes + 1 for name in tr.names: assert f"{name.replace('_', ' ')}" in ser assert "predictive entropy" in ser def test_the_score_is_not_on_the_page(app): """The demo is about the map, not the number; the API keeps the number.""" tr = make_trace() page = "".join(app.views(tr)) assert f"{tr.u_lens:.4f}" not in page and f"{tr.a_cone:.4f}" not in page assert "A_cone" not in page and "U-Lens" not in page assert tr.u_lens == tr.a_cone * tr.think_entropy # still there, untouched def test_a_truncated_trace_says_so_in_every_view(app): tr = make_trace(truncated=True, eot=19) # never closed: the budget case assert not tr.readout_valid head, tmap, ser = app.views(tr, signed=True) assert "never closed" in head and "never closed" in tmap and "never closed" in ser assert "ul-seg-answer" not in tmap # there is no answer to show assert "budget" in head and 'class="ul-trunc"' in head # the one status line, a warning assert "budget cut" in app.views(tr, panel="simplex")[2] def test_a_trace_without_a_thinking_block_reads_as_prompt_then_answer(app): """Thinking off: the template closes the block inside the prompt, `eot` lands on the prompt's last token, and every generated token is answer. No empty segment, no red rule, no thinking-block wording — and the axis values are shown like any others.""" tr = make_trace(prompt_len=6, eot=5) assert tr.think_len == 0 and not tr.truncated and not tr.readout_valid head, tmap, ser = app.views(tr, smooth=9) assert head == "" # a mode, not a warning assert "ul-seg-think" not in tmap and "ul-seg-answer" in tmap assert 'class="ul-eotmark none"' in tmap and "end of think" not in tmap assert "thinking block" not in tmap # a bare rule, no caption assert tmap.count('data-i="') == tr.n_tokens # every token is painted assert "prompt · answer" in ser and "prompt · thinking" not in ser assert 'class="eot"' not in ser and "ul-caption" not in ser assert "thinking block" not in ser and ser.count("") == 1 # the unit, once assert "nats" in ser assert ser.count('class="smooth"') == tr.n_axes + 1 # nothing greyed out sx = app.views(tr, smooth=9, panel="simplex")[2] assert 'class="eot"' not in sx and "Red ring" not in sx def test_signed_view_draws_a_zero_line_on_the_axis_rows_only(app): tr = make_trace() assert app.series_html(tr, signed=False).count('class="zero"') == 0 assert app.series_html(tr, signed=True).count('class="zero"') == tr.n_axes assert "points away" in app.token_map_html(tr, signed=True) assert "points away" not in app.token_map_html(tr, signed=False) # the value control already says "signed projections": the legend does not repeat it assert "signed projection" not in app.token_map_html(tr, signed=True) def test_each_channel_paints_the_map_in_its_own_wash(app): """One channel at a time, in that channel's single-hue ramp: the reader never has to tell two of the paper's colours apart.""" tr = make_trace() for ch in list(tr.names) + [app.ENTROPY]: html = app.token_map_html(tr, ch, smooth=1) w = app.WASH[ch] assert f"--w1:{w[0]}" in html and f"--w2:{w[1]}" in html and f"--w3:{w[2]}" in html assert f"{app.channel_label(ch)}" in html assert app.series_html(tr, ch).count('class="ul-row sel') == 1 # that row, only def test_washes_are_single_hue_light_to_dark(app): """Three steps per channel, ordered light to dark (validated separately with the dataviz validator in --ordinal mode; this keeps the dict from drifting).""" def lum(h): return sum(int(h[i:i + 2], 16) for i in (1, 3, 5)) for ch, steps in app.WASH.items(): assert len(steps) == 3 assert lum(steps[0]) > lum(steps[1]) > lum(steps[2]), ch def test_tint_bins_follow_the_range_and_never_divide_by_zero(app): b = app.tint_bins(np.linspace(0, 1, 21)) assert b.min() == 0 and b.max() == 3 and np.all(np.diff(b) >= 0) assert np.all(app.tint_bins(np.full(10, 2.0)) == 0) # constant: no tint assert np.all(app.tint_bins(np.full(5, np.nan)) == 0) # undefined: no tint s = app.tint_bins(np.array([-1.0, -0.1, 0.0, 0.5, 1.0]), signed=True) assert list(s) == [-1, 0, 0, 1, 3] # away / none / toward def test_block_selection_changes_the_values_not_the_trace(app): tr = make_trace() before = tr.field.copy() a = app.series_html(tr, block=tr.blocks[0]) b = app.series_html(tr, block=tr.blocks[-1]) assert a != b and _payload(a)["block"] == tr.blocks[0] assert f"block {tr.blocks[0]}" not in a # the control says which block assert app.header_html(tr, tr.blocks[0]) == "" # and there is no status line sa = app.simplex_html(tr, block=tr.blocks[0]) assert sa != app.simplex_html(tr, block=tr.blocks[-1]) and _payload(sa)["block"] == tr.blocks[0] np.testing.assert_array_equal(tr.field, before) with pytest.raises(KeyError): app.views(tr, block=999) def test_token_strings_are_escaped_everywhere(app): tr = make_trace(token_strs=[""] + [f"t{i}\n" for i in range(19)]) strs = app._hover_strs(tr) assert "" not in strs[0] and "<think>" in strs[0] assert "\n" not in strs[1] tmap = app.token_map_html(tr) assert "" not in tmap and "<think>" in tmap assert tmap.count('class="nl">↵\n') == 19 # a faint return mark, then a real break assert "¶" not in tmap # not a full-weight pilcrow mid-sentence def test_the_hover_payload_matches_the_series(app): tr = make_trace() d = _payload(app.series_html(tr, smooth=1)) assert d["T"] == tr.n_tokens and d["channels"] == list(tr.names) + [app.ENTROPY] assert d["seg"] == "".join(s[0] for s in tr.segments) assert len(d["raw"]) == tr.n_tokens and d["raw"][0][-1] is None # entropy[0] is NaN assert d["raw"][3][0] == pytest.approx(float(tr.components()[3, 0]), abs=1e-4) assert d["smooth"] == d["raw"] # w=1: nothing smoothed d9 = _payload(app.series_html(tr, smooth=9)) assert d9["w"] == 9 and d9["smooth"] != d9["raw"] def test_the_shipped_examples_load_without_a_model(app): paths = app.example_traces() assert len(paths) == 4 seen_truncated = False for p in paths: tr = runtime.load(p) head, tmap, ser = app.views(tr, smooth=9) assert tmap.count('data-i="') == tr.n_tokens assert ser.count('class="ul-row') == tr.n_axes + 1 seen_truncated |= "never closed" in head assert seen_truncated, "one example should be the never-closed thinking block" def test_the_default_example_is_the_paper_s_teaser_run(app): """The page opens on the run behind the paper's teaser figure: its prompt, thinking off, seed 0, a 1024-token budget. Listed first whatever the other files are called.""" paths = app.example_traces() assert paths[0].stem == app.DEFAULT_EXAMPLE tr = runtime.load(paths[0]) assert tr.prompt == "Answer briefly. Was Hamlet left-handed, and did he love Ophelia?" assert tr.config["thinking"] is False and tr.config["seed"] == 0 assert tr.config["max_new_tokens"] == 1024 assert tr.think_len == 0 and not tr.truncated and tr.n_tokens > tr.prompt_len # the pill is the prompt and nothing else assert app._example_label(paths[0]) == tr.prompt for p in paths: lab = app._example_label(p) assert "tokens" not in lab and "—" not in lab and len(lab) <= 64 def test_winner_lead_names_the_leading_axis_and_its_relative_lead(app): S = np.array([[1.0, 0.5, 0.2, 0.1], # a clear leader [0.4, 0.4, 0.1, 0.0], # a tie: no lead [0.3, 0.1, 2.0, 1.0], # a different leader, half ahead [np.nan, 1.0, 0.5, 0.2], # one undefined axis: it is never the leader [np.nan, np.nan, np.nan, 1.0]]) # no runner-up: no winner win, second, lead = app.winner_lead(S) assert win[0] == 0 and second[0] == 1 and lead[0] == pytest.approx(0.5) assert win[1] in (0, 1) and lead[1] == pytest.approx(0.0) assert win[2] == 2 and second[2] == 3 and lead[2] == pytest.approx(0.5) assert win[3] == 1 and second[3] == 2 and lead[3] == pytest.approx(0.5) assert win[4] == -1 and lead[4] == 0.0 # signed values: nothing points toward any axis, so nothing leads win_s, _, lead_s = app.winner_lead(np.array([[-0.2, -0.5, -0.1, -0.9], [0.6, -0.3, 0.1, 0.0]]), signed=True) assert list(win_s) == [-1, 0] and lead_s[0] == 0.0 assert lead_s[1] == pytest.approx(min(1.0, (0.6 - 0.1) / 0.6)) def test_the_all_four_mode_paints_each_token_in_its_leading_axis(app): """Colour is not the only carrier: the tint's strength is the lead over the runner-up, the legend lists the four axes, the hover payload names the winner, and every axis row in the series is the selected one.""" tr = make_trace(T=60, prompt_len=6, eot=50) tmap = app.token_map_html(tr, app.ALL, smooth=1) assert 'class="ul-map all"' in tmap and tmap.count('data-i="') == tr.n_tokens for k, name in enumerate(tr.names): assert f'class="chip ax{k}">{name.replace("_", " ")}' in tmap assert f"--c{k}:{app._rgb(app.WASH[name][2])}" in tmap assert "all four" in tmap and "lead over the runner-up" in tmap assert "none" not in tmap # the low/mid/high chips are not this legend R = app.axis_matrix(tr) win, _, lead = app.winner_lead(R) for i in range(tr.n_tokens): if lead[i] / app.LEAD_FULL >= 0.01: a = min(1.0, lead[i] / app.LEAD_FULL) assert f'class="t ax{win[i]}" data-i="{i}" style="--a:{a:.2f}"' in tmap else: assert f'class="t" data-i="{i}"' in tmap alphas = [float(a) for a in re.findall(r"--a:([\d.]+)", tmap)] assert alphas and max(alphas) <= 1.0 and min(alphas) >= 0.01 assert len(set(alphas)) > 3 # graded, not one flat colour ser = app.series_html(tr, app.ALL, smooth=1) assert ser.count('class="ul-row sel') == tr.n_axes d = _payload(ser) assert d["sel"] == app.ALL and d["win"] == [int(w) for w in win] assert len(d["second"]) == tr.n_tokens and d["lead"][3] == pytest.approx(lead[3], abs=1e-3) assert "D.sel === 'all'" in app.JS and "'no leading axis'" in app.JS def test_the_all_four_mode_is_smoothed_and_corrected_like_the_rows(app): """The map's winner at each token is the series' winner: same block, same smoothing.""" tr = make_trace(T=80, prompt_len=6, eot=70) tmap = app.token_map_html(tr, app.ALL, smooth=9, block=tr.blocks[1]) d = _payload(app.series_html(tr, app.ALL, smooth=9, block=tr.blocks[1])) S = np.array([[v for v in row[:tr.n_axes]] for row in d["smooth"]]) assert list(S.argmax(1)) == d["win"] for i, w in enumerate(d["win"]): if d["lead"][i] / app.LEAD_FULL >= 0.01: assert f'class="t ax{w}" data-i="{i}"' in tmap assert "smoothed over" not in tmap and f"block {tr.blocks[1]}" not in tmap # signed: a token whose every projection is negative has no leading axis and no tint tr2 = make_trace(T=12, prompt_len=2, eot=8) tr2.field[:, 5, :] = -1.0 tm = app.token_map_html(tr2, app.ALL, signed=True, smooth=1) assert 'class="t" data-i="5"' in tm def test_the_axis_pills_carry_their_axis_s_colour(app): """The control and the map share one language: each axis pill is outlined in its line colour and filled, when selected, with its darkest wash, on which ink is legible; the entropy pill takes its slate; "all four" stays neutral.""" css = app.CSS names = app._terms.categories() for pos, n in enumerate(names, start=1): assert (f".ul-axes-radio .wrap label:nth-child({pos}) " f"{{ --pc:{app.AXIS_COLOUR[n]}; --pf:{app.WASH[n][2]};") in css assert f"label:nth-child({len(names) + 1})" not in css # all four: no rule assert (f"label:nth-child({len(names) + 2}) {{ --pc:{app.ENTROPY_COLOUR}; " f"--pf:{app.WASH[app.ENTROPY][2]};") in css assert app.channel_label(app.ALL) == "all four" def test_no_view_restates_what_a_control_shows(app): """The control row says which block, which value, how much smoothing and whether the baseline is corrected; legends and captions carry only encodings a reader cannot infer.""" tr = make_trace(T=60, prompt_len=6, eot=50) for ch in list(tr.names) + [app.ALL, app.ENTROPY]: for signed in (False, True): tmap = app.token_map_html(tr, ch, signed, smooth=9, block=tr.blocks[1]) legend = re.search(r'
(.*?)
(.*?)

', sx).group(1) assert f"further {app.PATH_SMOOTH} tokens" in cap and "magnitude is dropped" in cap assert "block" not in cap and "rectified" not in cap and "baseline" not in cap def test_the_simplex_panel_draws_one_disc_per_token_in_its_leading_axis(app): """The third view: shares of the smoothed rectified components as barycentric coordinates, every token a disc coloured by its leader and sized by its lead, prompt tokens grey, the end-of-think token ringed; the frame is the four axes.""" from ulens import simplex as sx tr = make_trace(T=60, prompt_len=6, eot=50) html = app.simplex_html(tr, app.ALL, smooth=9) assert 'class="ul-simplex"' in html and html.count('{name.replace("_", " ")}' in html # colour and radius follow the leader and the lead of the doubly smoothed composition S, W, P = app.simplex_composition(tr, 9) win, _, lead = app.winner_lead(S) np.testing.assert_allclose(W.sum(1), 1.0) assert W.shape == (tr.n_tokens, tr.n_axes) and (W >= 0).all() assert (sx.boundary_fraction(P) <= 1.0 + 1e-9).all() # inside the frame found = re.findall(r' 1 # more than one leader # extra smoothing beyond the page's, declared; nothing escapes the tetrahedron when zoomed S9 = app.smooth_tokens(app.axis_matrix(tr).T, 9).T assert not np.allclose(S, S9) np.testing.assert_allclose(S, app.smooth_tokens(S9.T, app.PATH_SMOOTH).T, atol=1e-5) k = float(re.search(r"drawn ×([\d.]+) about the centre", html).group(1)) assert 1.0 < k <= app.SIMPLEX_ZOOM_MAX xs = [float(v) for v in re.findall(r'= 0).all() and (W_raw >= 0).all() html_signed = app.simplex_html(tr, signed=True, smooth=9, correction="centre") html_rect = app.simplex_html(tr, signed=False, smooth=9, correction="centre") assert "rectified components, not signed" in html_signed and "not signed" not in html_rect def discs(h): return re.findall(r'(.*?)
simplex" not in sx and "ul-maplegend" not in sx assert keys.count('class="k"') == 4 and keys.count("' in keys lo, hi = app.SIMPLEX_R assert f'' in keys assert f'r="{hi:g}"/>' in keys and keys.count('class="ko"') == 2 assert keys.count('class="kp"') == 1 and keys.count('class="ke"') == 1 for word in ("leading axis", "lead over the runner-up", "prompt", "end of think"): assert f"{word}" in keys, word assert "Red ring" not in sx # the ring is in the key, drawn assert "not signed" not in keys assert "rectified components, not signed" in app.simplex_html(tr, signed=True, smooth=9) # no thinking block: no ring is drawn, so none is in the key; a budget cut names itself sx0 = app.simplex_html(make_trace(prompt_len=6, eot=5), smooth=9) assert 'class="ke"' not in sx0 and "end of think" not in sx0 sxt = app.simplex_html(make_trace(truncated=True, eot=19), smooth=9) assert 'class="ke"' in sxt and "budget cut" in sxt # the all-four map: the four chips and a sample of the tint, faint to full, beside its # name; nothing that defines what a token is tmap = app.token_map_html(tr, app.ALL, smooth=9) legend = re.search(r'
(.*?)
lead over the runner-up' in legend assert legend.count('class="chip ax') == tr.n_axes and 'class="unit"' not in legend assert ".ul-maplegend .grad" in app.CSS and ".ul-keys .ko" in app.CSS # one axis: the swatch and name, then the wash's steps as chips, as before one = app.token_map_html(tr, tr.names[0], smooth=9) assert one.count('class="chip b') == 4 and not runon.search(one) # the hover reads the figure, not the key's samples assert "'.ul-simplex svg.fig'" in app.JS def test_the_link_bar_renders_only_what_is_configured(app, monkeypatch): """The paper, the blog and the code have built-in targets; each environment variable overrides its target, and set empty drops the link. A label with neither is not rendered — no dead href, no placeholder — and with nothing left the bar is absent and the head is exactly the bare one. "Try it locally" is the hosting command, shown as code, never a link, and has no built-in value while the package is not on PyPI.""" labels = [lab for lab, _var, _default in app.LINKS] assert labels == ["Read the paper", "Read the blog", "Try it locally", "See the code"] defaults = {var: d for _lab, var, d in app.LINKS} assert defaults["ULENS_LINK_PAPER"].startswith("https://arxiv.org/abs/") assert defaults["ULENS_LINK_BLOG"].startswith("https://") and defaults["ULENS_LINK_CODE"] assert defaults[app.LOCAL_ENV] == "" # not until the command works bar = app.links_html({}) assert bar.count("" not in bar for var in ("ULENS_LINK_PAPER", "ULENS_LINK_BLOG", "ULENS_LINK_CODE"): assert f'href="{defaults[var]}"' in bar assert bar.index("Read the paper") < bar.index("Read the blog") < bar.index("See the code") assert 'target="_blank" rel="noopener"' in bar and "coming soon" not in bar.lower() assert "ul-links" not in app.INTRO and "Try it locallypip install "ulens[app]"\n' 'ulens app') in cmd assert "") # build() reads the environment: the defaults, an override, or nothing at all def heads(demo): return [b.value for b in demo.blocks.values() if isinstance(b, gr.HTML) and b.value and "ul-links" in b.value] for var in defaults: monkeypatch.delenv(var, raising=False) h = heads(app.build()) assert len(h) == 1 and h[0].count("pip install ulens" in h[0] assert "Read the blog" not in h[0] and h[0].count("{panel}" not in right def test_the_mark_is_on_the_page_and_is_the_favicon(app): from ulens import simplex as sx assert sx.mark_svg() in app.INTRO and 'class="ul-mark"' in app.INTRO assert ".ul-mark svg" in app.CSS import inspect assert "favicon_path=str(_simplex.MARK_PATH)" in inspect.getsource(app.main) def test_the_separation_table_is_off_the_page_but_in_the_package(app): import ulens assert not hasattr(app, "separation_html") assert "Axis separation" not in app.CSS and "ul-cos" not in app.CSS assert callable(ulens.anatomy.separation) from pathlib import Path assert "Axis separation" not in Path(app.__file__).read_text()