Spaces:
Sleeping
Sleeping
One OpenRouter key; decision-model picker; single visible password box
Browse files
app.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
"""Jev Playground: put things in boxes, ask typed questions, read the probabilities.
|
| 2 |
|
| 3 |
-
|
|
|
|
| 4 |
localStorage (same scheme as blankaiarena and openrouter-claude), sent with each
|
| 5 |
request, and never stored or logged by the Space. On a private duplicate you can
|
| 6 |
-
instead set the secrets
|
| 7 |
"""
|
| 8 |
|
| 9 |
from __future__ import annotations
|
|
@@ -16,20 +17,26 @@ from concurrent.futures import ThreadPoolExecutor
|
|
| 16 |
|
| 17 |
import gradio as gr
|
| 18 |
|
| 19 |
-
from backends import BACKENDS, BackendError, OpenRouterEmulation, TypeSafeBackend
|
| 20 |
from presets import PRESETS, SCORE_LEVELS_DEFAULT
|
| 21 |
|
| 22 |
MAX_CANDIDATES = 10
|
| 23 |
-
|
| 24 |
-
ENV_KEYS = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
DEFAULT_PRESET = "Theories of consciousness"
|
| 26 |
|
| 27 |
|
| 28 |
# --- helpers -------------------------------------------------------------------
|
| 29 |
|
| 30 |
def resolve_key(backend: str, ts_key: str, or_key: str) -> str:
|
| 31 |
-
|
| 32 |
-
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def parse_candidates(boxes, n):
|
|
@@ -92,6 +99,11 @@ def bars_html(title, probs: dict, note=""):
|
|
| 92 |
f'{"".join(rows)}<div class="jp-note">{note}</div></div>')
|
| 93 |
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
def cell(v, txt):
|
| 96 |
return f'<td class="jp-cell" style="background:rgba(229,81,186,{0.08 + 0.72 * v:.2f})">{txt}</td>'
|
| 97 |
|
|
@@ -103,10 +115,10 @@ def refresh_models(backend, ts_key, or_key):
|
|
| 103 |
models = BACKENDS[backend].list_models(resolve_key(backend, ts_key, or_key))
|
| 104 |
except Exception as e:
|
| 105 |
raise gr.Error(f"Could not list models: {e}")
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
return gr.update(choices=models, value=default, info=
|
| 110 |
|
| 111 |
|
| 112 |
def load_preset(name):
|
|
@@ -133,9 +145,12 @@ def run_pick(backend, ts_key, or_key, model, context, n, question, *boxes):
|
|
| 133 |
except BackendError as e:
|
| 134 |
raise gr.Error(str(e))
|
| 135 |
a = res["answers"]["pick"]
|
| 136 |
-
note = (f"Top choice <b>{html.escape(a['choice'])}</b> ·
|
| 137 |
f"model {html.escape(res['model'])} · usage {html.escape(json.dumps(res['usage']))}")
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
| 139 |
|
| 140 |
|
| 141 |
def run_matrix(backend, ts_key, or_key, model, context, n, criteria_text, kind, levels_text, *boxes):
|
|
@@ -235,10 +250,11 @@ def run_single(backend, ts_key, or_key, model, state_text, qtype, instructions,
|
|
| 235 |
out = bars_html(instructions, {"yes": a["noul"], "no": 1 - a["noul"]},
|
| 236 |
f"noul = {a['noul']:.3f} (probability the answer is yes)")
|
| 237 |
elif a["type"] == "choice":
|
| 238 |
-
out = bars_html(instructions, a
|
| 239 |
else:
|
| 240 |
-
|
| 241 |
-
|
|
|
|
| 242 |
return out, redact(request), redact(res)
|
| 243 |
|
| 244 |
|
|
@@ -275,7 +291,7 @@ INTRO = """
|
|
| 275 |
Put things in boxes, ask **typed questions**, get **probabilities** back.
|
| 276 |
[Jev](https://docs.typesafe.ai) (TypeSafe) is a *System One* model: it doesn't write text, it answers
|
| 277 |
`noul` (P(yes)), `choice` (distribution over your options) or `score` (position on your rubric).
|
| 278 |
-
|
| 279 |
"""
|
| 280 |
|
| 281 |
HOW = """
|
|
@@ -306,18 +322,33 @@ prediction?", "is it consistent with finding X?") where the evidence is in the s
|
|
| 306 |
with weights *you* choose and can argue about.
|
| 307 |
|
| 308 |
### Backends
|
| 309 |
-
- **
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
"""
|
| 317 |
|
| 318 |
# Same per-device scheme as blankaiarena / openrouter-claude: localStorage, remembered by
|
| 319 |
# default, with a forget button. Every access is wrapped: storage can be blocked.
|
|
|
|
|
|
|
|
|
|
| 320 |
LOAD_KEYS_JS = """() => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
const get = k => { try { return localStorage.getItem(k) || ''; } catch { return ''; } };
|
| 322 |
return [get('jev.key.typesafe'), get('jev.key.openrouter'), get('jev.remember') !== '0'];
|
| 323 |
}"""
|
|
@@ -342,19 +373,20 @@ FORGET_KEYS_JS = """() => {
|
|
| 342 |
with gr.Blocks(title="Jev Playground") as demo:
|
| 343 |
gr.Markdown(INTRO)
|
| 344 |
with gr.Row(equal_height=True):
|
| 345 |
-
backend = gr.Radio(list(BACKENDS), value=
|
| 346 |
-
model = gr.Dropdown(
|
| 347 |
-
allow_custom_value=True, scale=2,
|
| 348 |
-
info=
|
| 349 |
refresh = gr.Button("↻ List models", scale=1)
|
| 350 |
-
with gr.Accordion("🔑
|
| 351 |
with gr.Row(equal_height=True):
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
type="password", scale=3)
|
| 355 |
with gr.Column(scale=2, min_width=180):
|
| 356 |
-
remember = gr.Checkbox(True, label="Remember my
|
| 357 |
-
forget = gr.Button("Forget saved
|
|
|
|
|
|
|
| 358 |
|
| 359 |
with gr.Tabs():
|
| 360 |
with gr.Tab("Compare candidates"):
|
|
@@ -436,7 +468,7 @@ with gr.Blocks(title="Jev Playground") as demo:
|
|
| 436 |
[s_out, s_req, s_res])
|
| 437 |
demo.load(load_preset, preset, [context, n, criteria, pick_q, *boxes]).then(show_boxes, n, boxes)
|
| 438 |
demo.load(None, None, [ts_key, or_key, remember], js=LOAD_KEYS_JS).then(
|
| 439 |
-
lambda ts, orr: gr.update(open=not
|
| 440 |
refresh_models, [backend, *keys], model)
|
| 441 |
|
| 442 |
|
|
|
|
| 1 |
"""Jev Playground: put things in boxes, ask typed questions, read the probabilities.
|
| 2 |
|
| 3 |
+
One OpenRouter key does everything: Jev (and the other decision models) are billed
|
| 4 |
+
to your OpenRouter credits. Paste it once per device: it is kept in this browser's
|
| 5 |
localStorage (same scheme as blankaiarena and openrouter-claude), sent with each
|
| 6 |
request, and never stored or logged by the Space. On a private duplicate you can
|
| 7 |
+
instead set the secrets OPENROUTER_API_KEY / TYPESAFE_API_KEY.
|
| 8 |
"""
|
| 9 |
|
| 10 |
from __future__ import annotations
|
|
|
|
| 17 |
|
| 18 |
import gradio as gr
|
| 19 |
|
| 20 |
+
from backends import BACKENDS, BackendError, OpenRouterDecisions, OpenRouterEmulation, TypeSafeBackend
|
| 21 |
from presets import PRESETS, SCORE_LEVELS_DEFAULT
|
| 22 |
|
| 23 |
MAX_CANDIDATES = 10
|
| 24 |
+
DEC_NAME, EMU_NAME, TS_NAME = OpenRouterDecisions.name, OpenRouterEmulation.name, TypeSafeBackend.name
|
| 25 |
+
ENV_KEYS = {"typesafe": "TYPESAFE_API_KEY", "openrouter": "OPENROUTER_API_KEY"}
|
| 26 |
+
MODEL_INFO = {
|
| 27 |
+
DEC_NAME: "OpenRouter models whose output is typed decisions, i.e. genuine classifiers.",
|
| 28 |
+
EMU_NAME: "{n} chat models that expose token logprobs, so they can imitate a classifier.",
|
| 29 |
+
TS_NAME: "Jev models served directly by TypeSafe.",
|
| 30 |
+
}
|
| 31 |
DEFAULT_PRESET = "Theories of consciousness"
|
| 32 |
|
| 33 |
|
| 34 |
# --- helpers -------------------------------------------------------------------
|
| 35 |
|
| 36 |
def resolve_key(backend: str, ts_key: str, or_key: str) -> str:
|
| 37 |
+
kind = BACKENDS[backend].key_kind
|
| 38 |
+
key = ts_key if kind == "typesafe" else or_key
|
| 39 |
+
return (key or "").strip() or os.environ.get(ENV_KEYS[kind], "")
|
| 40 |
|
| 41 |
|
| 42 |
def parse_candidates(boxes, n):
|
|
|
|
| 99 |
f'{"".join(rows)}<div class="jp-note">{note}</div></div>')
|
| 100 |
|
| 101 |
|
| 102 |
+
def conf_txt(a):
|
| 103 |
+
c = a.get("confidence")
|
| 104 |
+
return f"confidence {c:.2f}" if c is not None else "confidence not reported"
|
| 105 |
+
|
| 106 |
+
|
| 107 |
def cell(v, txt):
|
| 108 |
return f'<td class="jp-cell" style="background:rgba(229,81,186,{0.08 + 0.72 * v:.2f})">{txt}</td>'
|
| 109 |
|
|
|
|
| 115 |
models = BACKENDS[backend].list_models(resolve_key(backend, ts_key, or_key))
|
| 116 |
except Exception as e:
|
| 117 |
raise gr.Error(f"Could not list models: {e}")
|
| 118 |
+
values = [m[1] if isinstance(m, tuple) else m for m in models]
|
| 119 |
+
preferred = BACKENDS[backend].default_model
|
| 120 |
+
default = preferred if preferred in values else (values[0] if values else None)
|
| 121 |
+
return gr.update(choices=models, value=default, info=MODEL_INFO[backend].format(n=len(models)))
|
| 122 |
|
| 123 |
|
| 124 |
def load_preset(name):
|
|
|
|
| 145 |
except BackendError as e:
|
| 146 |
raise gr.Error(str(e))
|
| 147 |
a = res["answers"]["pick"]
|
| 148 |
+
note = (f"Top choice <b>{html.escape(a['choice'])}</b> · {conf_txt(a)} · "
|
| 149 |
f"model {html.escape(res['model'])} · usage {html.escape(json.dumps(res['usage']))}")
|
| 150 |
+
probs = a.get("probabilities") or {a["choice"]: 1.0}
|
| 151 |
+
if not a.get("probabilities"):
|
| 152 |
+
note += " · <b>this model returned only its pick, no distribution</b>"
|
| 153 |
+
return bars_html(question, probs, note), redact(request), redact(res)
|
| 154 |
|
| 155 |
|
| 156 |
def run_matrix(backend, ts_key, or_key, model, context, n, criteria_text, kind, levels_text, *boxes):
|
|
|
|
| 250 |
out = bars_html(instructions, {"yes": a["noul"], "no": 1 - a["noul"]},
|
| 251 |
f"noul = {a['noul']:.3f} (probability the answer is yes)")
|
| 252 |
elif a["type"] == "choice":
|
| 253 |
+
out = bars_html(instructions, a.get("probabilities") or {a["choice"]: 1.0}, conf_txt(a))
|
| 254 |
else:
|
| 255 |
+
legend = a.get("legend") or {str(i): lv for i, lv in enumerate(q["criteria"])}
|
| 256 |
+
probs = {f"{k}: {legend.get(k, '')}": v for k, v in (a.get("probabilities") or {}).items()}
|
| 257 |
+
out = bars_html(instructions, probs, f"score {a['score']:.2f} · {conf_txt(a)}")
|
| 258 |
return out, redact(request), redact(res)
|
| 259 |
|
| 260 |
|
|
|
|
| 291 |
Put things in boxes, ask **typed questions**, get **probabilities** back.
|
| 292 |
[Jev](https://docs.typesafe.ai) (TypeSafe) is a *System One* model: it doesn't write text, it answers
|
| 293 |
`noul` (P(yes)), `choice` (distribution over your options) or `score` (position on your rubric).
|
| 294 |
+
One **OpenRouter key** runs everything, billed to your OpenRouter credits. Paste it once per device and this browser remembers it.
|
| 295 |
"""
|
| 296 |
|
| 297 |
HOW = """
|
|
|
|
| 322 |
with weights *you* choose and can argue about.
|
| 323 |
|
| 324 |
### Backends
|
| 325 |
+
- **OpenRouter decision models (default)** — Jev plus the other models OpenRouter tags with the
|
| 326 |
+
output type `decisions` (Perplexity, Liquid, Cloudflare, Upstage, ...). The list comes live from
|
| 327 |
+
`GET /api/v1/models?output_modalities=decisions`; requests go to `POST /api/alpha/decisions`.
|
| 328 |
+
Billed to your OpenRouter credits. Some models leave out `confidence` or the full distribution;
|
| 329 |
+
confidence is then computed from the probabilities where possible.
|
| 330 |
+
- **OpenRouter chat LLM (logprob imitation)** — any chat model that exposes token logprobs, forced
|
| 331 |
+
to answer with one label token; we read the probability on each label. A comparison baseline:
|
| 332 |
+
chat models aren't trained for calibrated decisions, and each question is a separate call.
|
| 333 |
+
- **TypeSafe direct** — Jev from `api.typesafe.ai` with a TypeSafe key (Advanced, in the Keys panel).
|
| 334 |
+
|
| 335 |
+
(`typesafe/jev-router` is something else again: a router that uses Jev to pick which *chat* model
|
| 336 |
+
answers a chat request.)
|
| 337 |
"""
|
| 338 |
|
| 339 |
# Same per-device scheme as blankaiarena / openrouter-claude: localStorage, remembered by
|
| 340 |
# default, with a forget button. Every access is wrapped: storage can be blocked.
|
| 341 |
+
# Two password boxes side by side look like "password + confirm password" to phone password
|
| 342 |
+
# managers, which can take over the second one and block paste. So only one is visible, and
|
| 343 |
+
# both are marked as not-a-login-form for the managers that honour it.
|
| 344 |
LOAD_KEYS_JS = """() => {
|
| 345 |
+
for (const id of ['or-key', 'ts-key']) {
|
| 346 |
+
const el = document.querySelector(`#${id} input`);
|
| 347 |
+
if (!el) continue;
|
| 348 |
+
el.setAttribute('autocomplete', 'off');
|
| 349 |
+
for (const a of ['data-1p-ignore', 'data-lpignore', 'data-bwignore']) el.setAttribute(a, 'true');
|
| 350 |
+
el.setAttribute('data-form-type', 'other');
|
| 351 |
+
}
|
| 352 |
const get = k => { try { return localStorage.getItem(k) || ''; } catch { return ''; } };
|
| 353 |
return [get('jev.key.typesafe'), get('jev.key.openrouter'), get('jev.remember') !== '0'];
|
| 354 |
}"""
|
|
|
|
| 373 |
with gr.Blocks(title="Jev Playground") as demo:
|
| 374 |
gr.Markdown(INTRO)
|
| 375 |
with gr.Row(equal_height=True):
|
| 376 |
+
backend = gr.Radio(list(BACKENDS), value=DEC_NAME, label="Backend", scale=2, min_width=300)
|
| 377 |
+
model = gr.Dropdown([("TypeSafe: Jev Latest · ~typesafe/jev-latest", "~typesafe/jev-latest")],
|
| 378 |
+
value="~typesafe/jev-latest", label="Model", allow_custom_value=True, scale=2, min_width=300,
|
| 379 |
+
info=MODEL_INFO[DEC_NAME])
|
| 380 |
refresh = gr.Button("↻ List models", scale=1)
|
| 381 |
+
with gr.Accordion("🔑 Key — saved in this browser only", open=True) as keys_box:
|
| 382 |
with gr.Row(equal_height=True):
|
| 383 |
+
or_key = gr.Textbox(label="OpenRouter key (openrouter.ai/settings/keys)", type="password",
|
| 384 |
+
elem_id="or-key", scale=3)
|
|
|
|
| 385 |
with gr.Column(scale=2, min_width=180):
|
| 386 |
+
remember = gr.Checkbox(True, label="Remember my key on this device")
|
| 387 |
+
forget = gr.Button("Forget saved key", size="sm")
|
| 388 |
+
with gr.Accordion("Advanced: TypeSafe direct key (only for the TypeSafe direct backend)", open=False):
|
| 389 |
+
ts_key = gr.Textbox(label="TypeSafe key (console.typesafe.ai)", type="password", elem_id="ts-key")
|
| 390 |
|
| 391 |
with gr.Tabs():
|
| 392 |
with gr.Tab("Compare candidates"):
|
|
|
|
| 468 |
[s_out, s_req, s_res])
|
| 469 |
demo.load(load_preset, preset, [context, n, criteria, pick_q, *boxes]).then(show_boxes, n, boxes)
|
| 470 |
demo.load(None, None, [ts_key, or_key, remember], js=LOAD_KEYS_JS).then(
|
| 471 |
+
lambda ts, orr: gr.update(open=not orr), keys, keys_box).then(
|
| 472 |
refresh_models, [backend, *keys], model)
|
| 473 |
|
| 474 |
|