SoulFireMage commited on
Commit
a09cc89
·
verified ·
1 Parent(s): cb21495

One OpenRouter key; decision-model picker; single visible password box

Browse files
Files changed (1) hide show
  1. app.py +67 -35
app.py CHANGED
@@ -1,9 +1,10 @@
1
  """Jev Playground: put things in boxes, ask typed questions, read the probabilities.
2
 
3
- Bring your own key, pasted once per device: it is kept in this browser's
 
4
  localStorage (same scheme as blankaiarena and openrouter-claude), sent with each
5
  request, and never stored or logged by the Space. On a private duplicate you can
6
- instead set the secrets TYPESAFE_API_KEY / OPENROUTER_API_KEY.
7
  """
8
 
9
  from __future__ import annotations
@@ -16,20 +17,26 @@ from concurrent.futures import ThreadPoolExecutor
16
 
17
  import gradio as gr
18
 
19
- from backends import BACKENDS, BackendError, OpenRouterEmulation, TypeSafeBackend
20
  from presets import PRESETS, SCORE_LEVELS_DEFAULT
21
 
22
  MAX_CANDIDATES = 10
23
- TS_NAME, OR_NAME = TypeSafeBackend.name, OpenRouterEmulation.name
24
- ENV_KEYS = {TS_NAME: "TYPESAFE_API_KEY", OR_NAME: "OPENROUTER_API_KEY"}
 
 
 
 
 
25
  DEFAULT_PRESET = "Theories of consciousness"
26
 
27
 
28
  # --- helpers -------------------------------------------------------------------
29
 
30
  def resolve_key(backend: str, ts_key: str, or_key: str) -> str:
31
- key = ts_key if backend == TS_NAME else or_key
32
- return (key or "").strip() or os.environ.get(ENV_KEYS[backend], "")
 
33
 
34
 
35
  def parse_candidates(boxes, n):
@@ -92,6 +99,11 @@ def bars_html(title, probs: dict, note=""):
92
  f'{"".join(rows)}<div class="jp-note">{note}</div></div>')
93
 
94
 
 
 
 
 
 
95
  def cell(v, txt):
96
  return f'<td class="jp-cell" style="background:rgba(229,81,186,{0.08 + 0.72 * v:.2f})">{txt}</td>'
97
 
@@ -103,10 +115,10 @@ def refresh_models(backend, ts_key, or_key):
103
  models = BACKENDS[backend].list_models(resolve_key(backend, ts_key, or_key))
104
  except Exception as e:
105
  raise gr.Error(f"Could not list models: {e}")
106
- default = "jev-latest" if "jev-latest" in models else (models[0] if models else None)
107
- info = (f"{len(models)} models expose token logprobs, so they can imitate a classifier."
108
- if backend == OR_NAME else "Every TypeSafe model is a System One classifier.")
109
- return gr.update(choices=models, value=default, info=info)
110
 
111
 
112
  def load_preset(name):
@@ -133,9 +145,12 @@ def run_pick(backend, ts_key, or_key, model, context, n, question, *boxes):
133
  except BackendError as e:
134
  raise gr.Error(str(e))
135
  a = res["answers"]["pick"]
136
- note = (f"Top choice <b>{html.escape(a['choice'])}</b> · confidence {a['confidence']:.2f} · "
137
  f"model {html.escape(res['model'])} · usage {html.escape(json.dumps(res['usage']))}")
138
- return bars_html(question, a["probabilities"], note), redact(request), redact(res)
 
 
 
139
 
140
 
141
  def run_matrix(backend, ts_key, or_key, model, context, n, criteria_text, kind, levels_text, *boxes):
@@ -235,10 +250,11 @@ def run_single(backend, ts_key, or_key, model, state_text, qtype, instructions,
235
  out = bars_html(instructions, {"yes": a["noul"], "no": 1 - a["noul"]},
236
  f"noul = {a['noul']:.3f} (probability the answer is yes)")
237
  elif a["type"] == "choice":
238
- out = bars_html(instructions, a["probabilities"], f"confidence {a['confidence']:.2f}")
239
  else:
240
- probs = {f"{k}: {a['legend'][k]}": v for k, v in a["probabilities"].items()}
241
- out = bars_html(instructions, probs, f"score {a['score']:.2f} · confidence {a['confidence']:.2f}")
 
242
  return out, redact(request), redact(res)
243
 
244
 
@@ -275,7 +291,7 @@ INTRO = """
275
  Put things in boxes, ask **typed questions**, get **probabilities** back.
276
  [Jev](https://docs.typesafe.ai) (TypeSafe) is a *System One* model: it doesn't write text, it answers
277
  `noul` (P(yes)), `choice` (distribution over your options) or `score` (position on your rubric).
278
- Bring your own key: paste it once on each device and this browser remembers it (untick to keep it for this visit only).
279
  """
280
 
281
  HOW = """
@@ -306,18 +322,33 @@ prediction?", "is it consistent with finding X?") where the evidence is in the s
306
  with weights *you* choose and can argue about.
307
 
308
  ### Backends
309
- - **TypeSafe Jev (native)** — the real classifier at `api.typesafe.ai`. Key from
310
- [console.typesafe.ai](https://console.typesafe.ai). The model list comes from `GET /v1/models`.
311
- - **OpenRouter LLM (logprob emulation)** — any OpenRouter chat model that exposes token
312
- logprobs, forced to answer with one label token; we read the probability on each label.
313
- Good for comparison; ordinary LLMs aren't trained for calibrated decisions, and each question is a
314
- separate call. Note: OpenRouter's `typesafe/jev-router` is a *router* that picks an LLM for a chat
315
- request — it uses Jev internally but does not expose Jev's typed answers, so it isn't listed here.
 
 
 
 
 
316
  """
317
 
318
  # Same per-device scheme as blankaiarena / openrouter-claude: localStorage, remembered by
319
  # default, with a forget button. Every access is wrapped: storage can be blocked.
 
 
 
320
  LOAD_KEYS_JS = """() => {
 
 
 
 
 
 
 
321
  const get = k => { try { return localStorage.getItem(k) || ''; } catch { return ''; } };
322
  return [get('jev.key.typesafe'), get('jev.key.openrouter'), get('jev.remember') !== '0'];
323
  }"""
@@ -342,19 +373,20 @@ FORGET_KEYS_JS = """() => {
342
  with gr.Blocks(title="Jev Playground") as demo:
343
  gr.Markdown(INTRO)
344
  with gr.Row(equal_height=True):
345
- backend = gr.Radio(list(BACKENDS), value=TS_NAME, label="Backend", scale=2)
346
- model = gr.Dropdown(TypeSafeBackend.fallback_models, value="jev-latest", label="Model",
347
- allow_custom_value=True, scale=2,
348
- info="Every TypeSafe model is a System One classifier.")
349
  refresh = gr.Button("↻ List models", scale=1)
350
- with gr.Accordion("🔑 Keys — saved in this browser only", open=True) as keys_box:
351
  with gr.Row(equal_height=True):
352
- ts_key = gr.Textbox(label="TypeSafe key (console.typesafe.ai)", type="password", scale=3)
353
- or_key = gr.Textbox(label="OpenRouter key (openrouter.ai/keys) — optional, for comparison",
354
- type="password", scale=3)
355
  with gr.Column(scale=2, min_width=180):
356
- remember = gr.Checkbox(True, label="Remember my keys on this device")
357
- forget = gr.Button("Forget saved keys", size="sm")
 
 
358
 
359
  with gr.Tabs():
360
  with gr.Tab("Compare candidates"):
@@ -436,7 +468,7 @@ with gr.Blocks(title="Jev Playground") as demo:
436
  [s_out, s_req, s_res])
437
  demo.load(load_preset, preset, [context, n, criteria, pick_q, *boxes]).then(show_boxes, n, boxes)
438
  demo.load(None, None, [ts_key, or_key, remember], js=LOAD_KEYS_JS).then(
439
- lambda ts, orr: gr.update(open=not (ts or orr)), keys, keys_box).then(
440
  refresh_models, [backend, *keys], model)
441
 
442
 
 
1
  """Jev Playground: put things in boxes, ask typed questions, read the probabilities.
2
 
3
+ One OpenRouter key does everything: Jev (and the other decision models) are billed
4
+ to your OpenRouter credits. Paste it once per device: it is kept in this browser's
5
  localStorage (same scheme as blankaiarena and openrouter-claude), sent with each
6
  request, and never stored or logged by the Space. On a private duplicate you can
7
+ instead set the secrets OPENROUTER_API_KEY / TYPESAFE_API_KEY.
8
  """
9
 
10
  from __future__ import annotations
 
17
 
18
  import gradio as gr
19
 
20
+ from backends import BACKENDS, BackendError, OpenRouterDecisions, OpenRouterEmulation, TypeSafeBackend
21
  from presets import PRESETS, SCORE_LEVELS_DEFAULT
22
 
23
  MAX_CANDIDATES = 10
24
+ DEC_NAME, EMU_NAME, TS_NAME = OpenRouterDecisions.name, OpenRouterEmulation.name, TypeSafeBackend.name
25
+ ENV_KEYS = {"typesafe": "TYPESAFE_API_KEY", "openrouter": "OPENROUTER_API_KEY"}
26
+ MODEL_INFO = {
27
+ DEC_NAME: "OpenRouter models whose output is typed decisions, i.e. genuine classifiers.",
28
+ EMU_NAME: "{n} chat models that expose token logprobs, so they can imitate a classifier.",
29
+ TS_NAME: "Jev models served directly by TypeSafe.",
30
+ }
31
  DEFAULT_PRESET = "Theories of consciousness"
32
 
33
 
34
  # --- helpers -------------------------------------------------------------------
35
 
36
  def resolve_key(backend: str, ts_key: str, or_key: str) -> str:
37
+ kind = BACKENDS[backend].key_kind
38
+ key = ts_key if kind == "typesafe" else or_key
39
+ return (key or "").strip() or os.environ.get(ENV_KEYS[kind], "")
40
 
41
 
42
  def parse_candidates(boxes, n):
 
99
  f'{"".join(rows)}<div class="jp-note">{note}</div></div>')
100
 
101
 
102
+ def conf_txt(a):
103
+ c = a.get("confidence")
104
+ return f"confidence {c:.2f}" if c is not None else "confidence not reported"
105
+
106
+
107
  def cell(v, txt):
108
  return f'<td class="jp-cell" style="background:rgba(229,81,186,{0.08 + 0.72 * v:.2f})">{txt}</td>'
109
 
 
115
  models = BACKENDS[backend].list_models(resolve_key(backend, ts_key, or_key))
116
  except Exception as e:
117
  raise gr.Error(f"Could not list models: {e}")
118
+ values = [m[1] if isinstance(m, tuple) else m for m in models]
119
+ preferred = BACKENDS[backend].default_model
120
+ default = preferred if preferred in values else (values[0] if values else None)
121
+ return gr.update(choices=models, value=default, info=MODEL_INFO[backend].format(n=len(models)))
122
 
123
 
124
  def load_preset(name):
 
145
  except BackendError as e:
146
  raise gr.Error(str(e))
147
  a = res["answers"]["pick"]
148
+ note = (f"Top choice <b>{html.escape(a['choice'])}</b> · {conf_txt(a)} · "
149
  f"model {html.escape(res['model'])} · usage {html.escape(json.dumps(res['usage']))}")
150
+ probs = a.get("probabilities") or {a["choice"]: 1.0}
151
+ if not a.get("probabilities"):
152
+ note += " · <b>this model returned only its pick, no distribution</b>"
153
+ return bars_html(question, probs, note), redact(request), redact(res)
154
 
155
 
156
  def run_matrix(backend, ts_key, or_key, model, context, n, criteria_text, kind, levels_text, *boxes):
 
250
  out = bars_html(instructions, {"yes": a["noul"], "no": 1 - a["noul"]},
251
  f"noul = {a['noul']:.3f} (probability the answer is yes)")
252
  elif a["type"] == "choice":
253
+ out = bars_html(instructions, a.get("probabilities") or {a["choice"]: 1.0}, conf_txt(a))
254
  else:
255
+ legend = a.get("legend") or {str(i): lv for i, lv in enumerate(q["criteria"])}
256
+ probs = {f"{k}: {legend.get(k, '')}": v for k, v in (a.get("probabilities") or {}).items()}
257
+ out = bars_html(instructions, probs, f"score {a['score']:.2f} · {conf_txt(a)}")
258
  return out, redact(request), redact(res)
259
 
260
 
 
291
  Put things in boxes, ask **typed questions**, get **probabilities** back.
292
  [Jev](https://docs.typesafe.ai) (TypeSafe) is a *System One* model: it doesn't write text, it answers
293
  `noul` (P(yes)), `choice` (distribution over your options) or `score` (position on your rubric).
294
+ One **OpenRouter key** runs everything, billed to your OpenRouter credits. Paste it once per device and this browser remembers it.
295
  """
296
 
297
  HOW = """
 
322
  with weights *you* choose and can argue about.
323
 
324
  ### Backends
325
+ - **OpenRouter decision models (default)** — Jev plus the other models OpenRouter tags with the
326
+ output type `decisions` (Perplexity, Liquid, Cloudflare, Upstage, ...). The list comes live from
327
+ `GET /api/v1/models?output_modalities=decisions`; requests go to `POST /api/alpha/decisions`.
328
+ Billed to your OpenRouter credits. Some models leave out `confidence` or the full distribution;
329
+ confidence is then computed from the probabilities where possible.
330
+ - **OpenRouter chat LLM (logprob imitation)** — any chat model that exposes token logprobs, forced
331
+ to answer with one label token; we read the probability on each label. A comparison baseline:
332
+ chat models aren't trained for calibrated decisions, and each question is a separate call.
333
+ - **TypeSafe direct** — Jev from `api.typesafe.ai` with a TypeSafe key (Advanced, in the Keys panel).
334
+
335
+ (`typesafe/jev-router` is something else again: a router that uses Jev to pick which *chat* model
336
+ answers a chat request.)
337
  """
338
 
339
  # Same per-device scheme as blankaiarena / openrouter-claude: localStorage, remembered by
340
  # default, with a forget button. Every access is wrapped: storage can be blocked.
341
+ # Two password boxes side by side look like "password + confirm password" to phone password
342
+ # managers, which can take over the second one and block paste. So only one is visible, and
343
+ # both are marked as not-a-login-form for the managers that honour it.
344
  LOAD_KEYS_JS = """() => {
345
+ for (const id of ['or-key', 'ts-key']) {
346
+ const el = document.querySelector(`#${id} input`);
347
+ if (!el) continue;
348
+ el.setAttribute('autocomplete', 'off');
349
+ for (const a of ['data-1p-ignore', 'data-lpignore', 'data-bwignore']) el.setAttribute(a, 'true');
350
+ el.setAttribute('data-form-type', 'other');
351
+ }
352
  const get = k => { try { return localStorage.getItem(k) || ''; } catch { return ''; } };
353
  return [get('jev.key.typesafe'), get('jev.key.openrouter'), get('jev.remember') !== '0'];
354
  }"""
 
373
  with gr.Blocks(title="Jev Playground") as demo:
374
  gr.Markdown(INTRO)
375
  with gr.Row(equal_height=True):
376
+ backend = gr.Radio(list(BACKENDS), value=DEC_NAME, label="Backend", scale=2, min_width=300)
377
+ model = gr.Dropdown([("TypeSafe: Jev Latest · ~typesafe/jev-latest", "~typesafe/jev-latest")],
378
+ value="~typesafe/jev-latest", label="Model", allow_custom_value=True, scale=2, min_width=300,
379
+ info=MODEL_INFO[DEC_NAME])
380
  refresh = gr.Button("↻ List models", scale=1)
381
+ with gr.Accordion("🔑 Key — saved in this browser only", open=True) as keys_box:
382
  with gr.Row(equal_height=True):
383
+ or_key = gr.Textbox(label="OpenRouter key (openrouter.ai/settings/keys)", type="password",
384
+ elem_id="or-key", scale=3)
 
385
  with gr.Column(scale=2, min_width=180):
386
+ remember = gr.Checkbox(True, label="Remember my key on this device")
387
+ forget = gr.Button("Forget saved key", size="sm")
388
+ with gr.Accordion("Advanced: TypeSafe direct key (only for the TypeSafe direct backend)", open=False):
389
+ ts_key = gr.Textbox(label="TypeSafe key (console.typesafe.ai)", type="password", elem_id="ts-key")
390
 
391
  with gr.Tabs():
392
  with gr.Tab("Compare candidates"):
 
468
  [s_out, s_req, s_res])
469
  demo.load(load_preset, preset, [context, n, criteria, pick_q, *boxes]).then(show_boxes, n, boxes)
470
  demo.load(None, None, [ts_key, or_key, remember], js=LOAD_KEYS_JS).then(
471
+ lambda ts, orr: gr.update(open=not orr), keys, keys_box).then(
472
  refresh_models, [backend, *keys], model)
473
 
474