Spaces:
Sleeping
Sleeping
Upload folder using huggingface_hub
Browse files- README.md +44 -7
- app.py +102 -0
- jev_core.py +187 -0
- requirements.txt +4 -0
README.md
CHANGED
|
@@ -1,13 +1,50 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
|
|
|
|
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: JEV-9B Decision API
|
| 3 |
+
emoji: ⚖️
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.28.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
python_version: "3.12"
|
| 10 |
+
startup_duration_timeout: 1h
|
| 11 |
pinned: false
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# JEV-9B Decision API
|
| 15 |
+
|
| 16 |
+
A minimal, machine-friendly wrapper over **System 1** of
|
| 17 |
+
[`autotrust/JEV-9B`](https://huggingface.co/autotrust/JEV-9B) — AutoTrust AI's open
|
| 18 |
+
*decision* model (Apache-2.0), built with the Blocks-of-Experts recipe on a frozen
|
| 19 |
+
Qwen3.5-9B backbone.
|
| 20 |
+
|
| 21 |
+
System 1 answers a **typed question** in a *single prefill pass* (nothing is generated)
|
| 22 |
+
and returns a **calibrated probability distribution**:
|
| 23 |
+
|
| 24 |
+
- `noul` — "Is this true?" → `[P(false), P(true)]`
|
| 25 |
+
- `choice` — which of 2–16 options → one probability per option
|
| 26 |
+
- `score` — where on an ordered 0–5 scale → distribution over six levels
|
| 27 |
+
|
| 28 |
+
## API
|
| 29 |
+
|
| 30 |
+
A single Gradio endpoint, `/decide`, taking `kind`, `state`, `question`, `options`
|
| 31 |
+
(comma-separated, `choice` only) and returning JSON:
|
| 32 |
+
|
| 33 |
+
```json
|
| 34 |
+
{"kind":"choice","distribution":{"a":0.53,"b":0.33,"c":0.14},"best":"a",
|
| 35 |
+
"confidence":0.53,"state_tokens":18,"truncated":false}
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
This Space exists to give local agents (Hermes) one clean JSON call, rather than the
|
| 39 |
+
multi-question HTML UI of the upstream demo. Inference core (`jev_core.py`) is the
|
| 40 |
+
model authors' documented `transformers` + `peft` path, reused verbatim.
|
| 41 |
+
|
| 42 |
+
### Provenance & limits
|
| 43 |
+
|
| 44 |
+
`jev_core.py` is reused unmodified from the authors' official demo Space
|
| 45 |
+
([`autotrust/jev-9b-decision-demo`](https://huggingface.co/spaces/autotrust/jev-9b-decision-demo)),
|
| 46 |
+
which is Apache-2.0. Model card caveats still apply: System 1 mirrors its teacher
|
| 47 |
+
**including its mistakes** (multi-hop reasoning, arithmetic, dates, counting, adversarial
|
| 48 |
+
inputs), reads English best, uses the first 1,024 tokens of state, and is **not for
|
| 49 |
+
high-stakes decisions** — use confidence gating and route low-confidence decisions
|
| 50 |
+
elsewhere. Runs on ZeroGPU (`zero-a10g`).
|
app.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JEV-9B Decision API — one clean JSON call over System 1.
|
| 2 |
+
|
| 3 |
+
Inference core (`jev_core.py`) is the model authors' documented `transformers` + `peft`
|
| 4 |
+
path, reused verbatim from the official demo Space (Apache-2.0). This file only wraps it
|
| 5 |
+
in a single machine-friendly Gradio endpoint (`/decide`) that returns JSON, instead of the
|
| 6 |
+
upstream multi-question HTML UI, so a local agent can call it headlessly.
|
| 7 |
+
|
| 8 |
+
ZeroGPU rules honoured here:
|
| 9 |
+
* `import spaces` MUST precede torch / transformers / any CUDA-touching import
|
| 10 |
+
(jev_core imports torch, so `spaces` is imported before it).
|
| 11 |
+
* models are placed on `cuda` at MODULE level (inside jev_core, at import) — not
|
| 12 |
+
inside the `@spaces.GPU` function.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
|
| 17 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 18 |
+
os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")
|
| 19 |
+
|
| 20 |
+
import spaces # noqa: E402 -- MUST precede torch / transformers / any CUDA-touching import
|
| 21 |
+
|
| 22 |
+
import json # noqa: E402
|
| 23 |
+
|
| 24 |
+
import gradio as gr # noqa: E402
|
| 25 |
+
|
| 26 |
+
import jev_core as jc # noqa: E402
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _clean_options(raw) -> list:
|
| 30 |
+
"""Accept a comma/newline-separated string (or a list) -> list of option strings."""
|
| 31 |
+
if isinstance(raw, (list, tuple)):
|
| 32 |
+
items = [str(o).strip() for o in raw]
|
| 33 |
+
else:
|
| 34 |
+
items = [o.strip() for o in str(raw or "").replace("\n", ",").split(",")]
|
| 35 |
+
return [o for o in items if o]
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@spaces.GPU(duration=45)
|
| 39 |
+
def decide(kind: str, state: str, question: str, options: str = "") -> str:
|
| 40 |
+
"""Answer ONE typed System 1 decision; return calibrated probabilities as JSON.
|
| 41 |
+
|
| 42 |
+
kind: "noul" | "choice" | "score"
|
| 43 |
+
state: the text to reason about (truncated to 1024 tokens, head 60% / tail 40%)
|
| 44 |
+
question: the decision question
|
| 45 |
+
options: comma-separated options (choice only, 2-16); ignored for noul/score
|
| 46 |
+
"""
|
| 47 |
+
kind = (kind or "").strip()
|
| 48 |
+
state = (state or "").strip()
|
| 49 |
+
question = (question or "").strip()
|
| 50 |
+
|
| 51 |
+
if kind not in jc.KINDS:
|
| 52 |
+
return json.dumps({"error": f"kind must be one of {list(jc.KINDS)}"})
|
| 53 |
+
if not state:
|
| 54 |
+
return json.dumps({"error": "state is required"})
|
| 55 |
+
if not question:
|
| 56 |
+
return json.dumps({"error": "question is required"})
|
| 57 |
+
|
| 58 |
+
opts = _clean_options(options)
|
| 59 |
+
if kind == "choice":
|
| 60 |
+
if not 2 <= len(opts) <= 16:
|
| 61 |
+
return json.dumps({"error": "choice needs 2-16 comma-separated options"})
|
| 62 |
+
if len(set(opts)) != len(opts):
|
| 63 |
+
return json.dumps({"error": "choice options must be unique"})
|
| 64 |
+
else:
|
| 65 |
+
opts = None
|
| 66 |
+
|
| 67 |
+
state_used, truncated, n_tok = jc.truncate_state(state)
|
| 68 |
+
dist = jc.decide(kind, state_used, question, opts)
|
| 69 |
+
best = max(dist, key=dist.get)
|
| 70 |
+
return json.dumps(
|
| 71 |
+
{
|
| 72 |
+
"kind": kind,
|
| 73 |
+
"distribution": dist,
|
| 74 |
+
"best": best,
|
| 75 |
+
"confidence": dist[best],
|
| 76 |
+
"state_tokens": n_tok,
|
| 77 |
+
"truncated": truncated,
|
| 78 |
+
}
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
with gr.Blocks(title="JEV-9B Decision API") as demo:
|
| 83 |
+
gr.Markdown(
|
| 84 |
+
"## JEV-9B System 1 — typed decisions in one prefill pass\n"
|
| 85 |
+
"`noul` (yes/no) · `choice` (2–16 options) · `score` (0–5). "
|
| 86 |
+
"Returns a calibrated probability distribution as JSON."
|
| 87 |
+
)
|
| 88 |
+
with gr.Row():
|
| 89 |
+
kind_in = gr.Dropdown(list(jc.KINDS), value="noul", label="kind")
|
| 90 |
+
state_in = gr.Textbox(label="state", lines=3)
|
| 91 |
+
question_in = gr.Textbox(label="question")
|
| 92 |
+
options_in = gr.Textbox(label="options (comma-separated; choice only)")
|
| 93 |
+
out_json = gr.Code(label="decision JSON", language="json")
|
| 94 |
+
btn = gr.Button("Decide", variant="primary")
|
| 95 |
+
btn.click(
|
| 96 |
+
decide,
|
| 97 |
+
inputs=[kind_in, state_in, question_in, options_in],
|
| 98 |
+
outputs=[out_json],
|
| 99 |
+
api_name="decide",
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
demo.launch()
|
jev_core.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JEV-9B inference core, ported 1:1 from the model authors' documented `transformers` +
|
| 2 |
+
`peft` path (https://huggingface.co/autotrust/JEV-9B -> "Other ways to run it").
|
| 3 |
+
|
| 4 |
+
The Blocks-of-Experts architecture:
|
| 5 |
+
* backbone = frozen Qwen3.5-9B text tower (bit-identical to the release)
|
| 6 |
+
* System 1 = LoRA r=16 (40.1M, shipped unmerged in `adapter/`) + a 24-slot fp32
|
| 7 |
+
decision head (`head.safetensors`) initialised from `lm_head` verbalizer rows
|
| 8 |
+
* System 2 = the original `lm_head`; used with the adapter off (pristine Qwen3.5-9B)
|
| 9 |
+
|
| 10 |
+
System 1 answers a typed question (`noul` yes/no, `choice` over 2-16 options, `score` on a
|
| 11 |
+
0-5 scale) in ONE prefill pass and returns a calibrated probability distribution. The
|
| 12 |
+
decision prompt follows the authors' "bare-v1" template; the head is applied to the last
|
| 13 |
+
token's post-norm hidden state, divided by the per-kind calibration temperature.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from huggingface_hub import hf_hub_download
|
| 21 |
+
from peft import PeftModel
|
| 22 |
+
from safetensors.torch import load_file
|
| 23 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 24 |
+
|
| 25 |
+
REPO = "autotrust/JEV-9B"
|
| 26 |
+
LETTERS = "ABCDEFGHIJKLMNOP"
|
| 27 |
+
KINDS = ("noul", "choice", "score")
|
| 28 |
+
MAX_STATE_TOKENS = 1024 # the authors truncate long inputs to 1,024 tokens (head 60% / tail 40%)
|
| 29 |
+
DEVICE = os.environ.get("JEV_DEVICE", "cuda")
|
| 30 |
+
# Optional local-dir override (used only for offline verification of the wiring).
|
| 31 |
+
_LOCAL = os.environ.get("JEV_MODEL_PATH")
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _file(name: str) -> str:
|
| 35 |
+
"""Resolve a model-repo file to a local path, falling back to the Hub."""
|
| 36 |
+
if _LOCAL:
|
| 37 |
+
p = os.path.join(_LOCAL, name)
|
| 38 |
+
if os.path.exists(p):
|
| 39 |
+
return p
|
| 40 |
+
return hf_hub_download(REPO, name)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
_SRC = _LOCAL or REPO
|
| 44 |
+
|
| 45 |
+
# ---------------------------------------------------------------------------
|
| 46 |
+
# Model loading. On ZeroGPU the final `_peft.to(DEVICE)` is intercepted at
|
| 47 |
+
# startup: tensors are packed to disk and streamed into VRAM on the first
|
| 48 |
+
# @spaces.GPU entry, so the load cost is never paid inside a request.
|
| 49 |
+
#
|
| 50 |
+
# Everything is built on CPU first, then moved once:
|
| 51 |
+
# * peft's default would be `torch_device=infer_device()` -> "cuda", making
|
| 52 |
+
# safetensors read the adapter straight onto the GPU. That bypasses the
|
| 53 |
+
# packer and crashes the GPU-less main process ("No CUDA GPUs are
|
| 54 |
+
# available"). `torch_device="cpu"` keeps the read on the CPU.
|
| 55 |
+
# * one `.to(DEVICE)` at the end moves every tensor (backbone, LoRA, head)
|
| 56 |
+
# through the standard machinery.
|
| 57 |
+
# ---------------------------------------------------------------------------
|
| 58 |
+
_tok = AutoTokenizer.from_pretrained(_SRC)
|
| 59 |
+
_base = AutoModelForCausalLM.from_pretrained(_SRC, dtype=torch.bfloat16)
|
| 60 |
+
|
| 61 |
+
# Keep the adapter UNMERGED so System 2 (generation) runs with the adapter
|
| 62 |
+
# disabled -- `with _peft.disable_adapter():` -- i.e. pristine Qwen3.5-9B.
|
| 63 |
+
_peft = PeftModel.from_pretrained(_base, _SRC, subfolder="adapter", torch_device="cpu")
|
| 64 |
+
|
| 65 |
+
# System 1 head: fp32 linear, 4096 -> 24 slots.
|
| 66 |
+
_HEAD = torch.nn.Linear(4096, 24, dtype=torch.float32)
|
| 67 |
+
_head_sd = load_file(_file("head.safetensors"))
|
| 68 |
+
with torch.no_grad():
|
| 69 |
+
_HEAD.weight.copy_(_head_sd["proj.weight"])
|
| 70 |
+
_HEAD.bias.copy_(_head_sd["proj.bias"])
|
| 71 |
+
_HEAD.eval()
|
| 72 |
+
|
| 73 |
+
_JUDGE_CFG = json.load(open(_file("judge_config.json")))
|
| 74 |
+
_CALIB = json.load(open(_file("calibration.json")))["per_kind"]
|
| 75 |
+
SLOT_RANGES = {k: tuple(v) for k, v in _JUDGE_CFG["slots"]["ranges"].items()}
|
| 76 |
+
TEMPERATURES = {k: float(_CALIB[k]) for k in KINDS}
|
| 77 |
+
|
| 78 |
+
# Module handles: PeftModel -> LoraModel -> Qwen3_5ForCausalLM -> Qwen3_5TextModel.
|
| 79 |
+
_CAUSAL = _peft.base_model.model # the Qwen3_5ForCausalLM with LoRA modules inside
|
| 80 |
+
_BACKBONE = _CAUSAL.model # the text tower (LoRA active by default)
|
| 81 |
+
|
| 82 |
+
# Single move of the whole tree (backbone + LoRA + head) to the compute device.
|
| 83 |
+
_peft.to(DEVICE)
|
| 84 |
+
_HEAD.to(DEVICE)
|
| 85 |
+
|
| 86 |
+
# ---------------------------------------------------------------------------
|
| 87 |
+
# Prompt template -- exactly the authors' `bare-v1` decision format.
|
| 88 |
+
# ---------------------------------------------------------------------------
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def build_decision_prompt(kind: str, state: str, question: str, options) -> tuple:
|
| 92 |
+
"""Return (prompt_text, option_labels) in the authors' `bare-v1` format."""
|
| 93 |
+
if kind == "noul":
|
| 94 |
+
options = ["false", "true"]
|
| 95 |
+
elif kind == "score":
|
| 96 |
+
options = [str(i) for i in range(6)]
|
| 97 |
+
options = list(options)
|
| 98 |
+
if kind == "choice":
|
| 99 |
+
lines = [f"{LETTERS[i]}) {o}" for i, o in enumerate(options)]
|
| 100 |
+
else:
|
| 101 |
+
lines = options
|
| 102 |
+
prompt = (
|
| 103 |
+
f"[kind] {kind}\n[state] {state}\n[question] {question}\n[options]\n"
|
| 104 |
+
+ "\n".join(lines)
|
| 105 |
+
+ "\n[decision]:"
|
| 106 |
+
)
|
| 107 |
+
return prompt, options
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def truncate_state(state: str) -> tuple:
|
| 111 |
+
"""Head-60% / tail-40% truncation of the state to `MAX_STATE_TOKENS` tokens.
|
| 112 |
+
|
| 113 |
+
Returns (state_used, was_truncated, n_tokens).
|
| 114 |
+
"""
|
| 115 |
+
ids = _tok(state, add_special_tokens=False)["input_ids"]
|
| 116 |
+
if len(ids) <= MAX_STATE_TOKENS:
|
| 117 |
+
return state, False, len(ids)
|
| 118 |
+
keep_head = int(MAX_STATE_TOKENS * 0.6)
|
| 119 |
+
keep_tail = MAX_STATE_TOKENS - keep_head
|
| 120 |
+
return _tok.decode(ids[:keep_head] + ids[-keep_tail:], skip_special_tokens=True), True, len(ids)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def decide(kind: str, state: str, question: str, options=None) -> dict:
|
| 124 |
+
"""Run ONE System 1 typed decision; return the calibrated probability distribution.
|
| 125 |
+
|
| 126 |
+
This mirrors the model card's reference `decide()` byte for byte: one prefill pass
|
| 127 |
+
through the backbone with the LoRA active, the 24-slot head applied to the last
|
| 128 |
+
token's hidden state, per-kind temperature, softmax over the active slots.
|
| 129 |
+
"""
|
| 130 |
+
if kind not in KINDS:
|
| 131 |
+
raise ValueError(f"unknown kind {kind!r}")
|
| 132 |
+
if kind == "choice":
|
| 133 |
+
if not isinstance(options, (list, tuple)) or not 2 <= len(options) <= 16:
|
| 134 |
+
raise ValueError("`choice` needs between 2 and 16 options")
|
| 135 |
+
if len(set(options)) != len(options):
|
| 136 |
+
raise ValueError("`choice` options must be unique")
|
| 137 |
+
prompt, labels = build_decision_prompt(kind, state, question, options)
|
| 138 |
+
device = _BACKBONE.embed_tokens.weight.device # resolve inside the GPU call
|
| 139 |
+
ids = _tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
|
| 140 |
+
with torch.no_grad():
|
| 141 |
+
h = _BACKBONE(**ids).last_hidden_state[0, -1].float()
|
| 142 |
+
z = _HEAD(h.to(_HEAD.weight.device)) # [24] fp32 logits
|
| 143 |
+
lo = SLOT_RANGES[kind][0]
|
| 144 |
+
z = z[lo : lo + len(labels)] / TEMPERATURES[kind]
|
| 145 |
+
p = torch.softmax(z, dim=0)
|
| 146 |
+
return {label: float(v) for label, v in zip(labels, p.tolist())}
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def n_state_tokens(state: str) -> int:
|
| 150 |
+
return len(_tok(state, add_special_tokens=False)["input_ids"])
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
# ---------------------------------------------------------------------------
|
| 154 |
+
# System 2 -- ordinary generation from the pristine base model (adapter off).
|
| 155 |
+
# ---------------------------------------------------------------------------
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def build_escalation_prompt(state: str, question: str, options) -> str:
|
| 159 |
+
"""The model card's confidence-gated escalation prompt (System 1 -> System 2)."""
|
| 160 |
+
opts = [str(o) for o in options]
|
| 161 |
+
if opts == ["false", "true"]:
|
| 162 |
+
tail = "Answer `true` or `false` on the last line."
|
| 163 |
+
else:
|
| 164 |
+
tail = "Options: " + "; ".join(opts) + "\nThink it through, then give exactly one option on the last line."
|
| 165 |
+
return f"{state}\n\nQuestion: {question}\n{tail}"
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def generate(state: str, question: str, options, max_new_tokens: int = 512,
|
| 169 |
+
enable_thinking: bool = True) -> tuple:
|
| 170 |
+
"""System 2: generate a reasoned answer with the adapter OFF -- pristine Qwen3.5-9B.
|
| 171 |
+
|
| 172 |
+
Returns (text, completed): `completed` is False when generation hit the token
|
| 173 |
+
budget mid-sentence (the model is verbose in thinking mode), which the UI needs
|
| 174 |
+
to avoid presenting a cut-off trace as a finished answer.
|
| 175 |
+
"""
|
| 176 |
+
prompt = build_escalation_prompt(state, question, options)
|
| 177 |
+
msgs = [{"role": "user", "content": prompt}]
|
| 178 |
+
enc = _tok.apply_chat_template(
|
| 179 |
+
msgs, add_generation_prompt=True, enable_thinking=enable_thinking,
|
| 180 |
+
return_tensors="pt", return_dict=True,
|
| 181 |
+
).to(_BACKBONE.embed_tokens.weight.device)
|
| 182 |
+
n_prompt = enc["input_ids"].shape[1]
|
| 183 |
+
with torch.no_grad(), _peft.disable_adapter():
|
| 184 |
+
out = _CAUSAL.generate(**enc, max_new_tokens=int(max_new_tokens), do_sample=False)
|
| 185 |
+
text = _tok.decode(out[0, n_prompt:], skip_special_tokens=True)
|
| 186 |
+
completed = out.shape[1] - n_prompt < int(max_new_tokens) # stopped before the cap
|
| 187 |
+
return text.strip(), completed
|
requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
transformers==5.16.1
|
| 2 |
+
peft==0.21.0
|
| 3 |
+
accelerate
|
| 4 |
+
safetensors
|