StrangerInAStrangeLand commited on
Commit
db45a11
·
verified ·
1 Parent(s): 35756da

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. README.md +44 -7
  2. app.py +102 -0
  3. jev_core.py +187 -0
  4. requirements.txt +4 -0
README.md CHANGED
@@ -1,13 +1,50 @@
1
  ---
2
- title: Jev Decision
3
- emoji: 🐠
4
- colorFrom: green
5
- colorTo: green
6
  sdk: gradio
7
- sdk_version: 6.30.0
8
- python_version: '3.12'
9
  app_file: app.py
 
 
10
  pinned: false
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: JEV-9B Decision API
3
+ emoji: ⚖️
4
+ colorFrom: blue
5
+ colorTo: gray
6
  sdk: gradio
7
+ sdk_version: 6.28.0
 
8
  app_file: app.py
9
+ python_version: "3.12"
10
+ startup_duration_timeout: 1h
11
  pinned: false
12
  ---
13
 
14
+ # JEV-9B Decision API
15
+
16
+ A minimal, machine-friendly wrapper over **System 1** of
17
+ [`autotrust/JEV-9B`](https://huggingface.co/autotrust/JEV-9B) — AutoTrust AI's open
18
+ *decision* model (Apache-2.0), built with the Blocks-of-Experts recipe on a frozen
19
+ Qwen3.5-9B backbone.
20
+
21
+ System 1 answers a **typed question** in a *single prefill pass* (nothing is generated)
22
+ and returns a **calibrated probability distribution**:
23
+
24
+ - `noul` — "Is this true?" → `[P(false), P(true)]`
25
+ - `choice` — which of 2–16 options → one probability per option
26
+ - `score` — where on an ordered 0–5 scale → distribution over six levels
27
+
28
+ ## API
29
+
30
+ A single Gradio endpoint, `/decide`, taking `kind`, `state`, `question`, `options`
31
+ (comma-separated, `choice` only) and returning JSON:
32
+
33
+ ```json
34
+ {"kind":"choice","distribution":{"a":0.53,"b":0.33,"c":0.14},"best":"a",
35
+ "confidence":0.53,"state_tokens":18,"truncated":false}
36
+ ```
37
+
38
+ This Space exists to give local agents (Hermes) one clean JSON call, rather than the
39
+ multi-question HTML UI of the upstream demo. Inference core (`jev_core.py`) is the
40
+ model authors' documented `transformers` + `peft` path, reused verbatim.
41
+
42
+ ### Provenance & limits
43
+
44
+ `jev_core.py` is reused unmodified from the authors' official demo Space
45
+ ([`autotrust/jev-9b-decision-demo`](https://huggingface.co/spaces/autotrust/jev-9b-decision-demo)),
46
+ which is Apache-2.0. Model card caveats still apply: System 1 mirrors its teacher
47
+ **including its mistakes** (multi-hop reasoning, arithmetic, dates, counting, adversarial
48
+ inputs), reads English best, uses the first 1,024 tokens of state, and is **not for
49
+ high-stakes decisions** — use confidence gating and route low-confidence decisions
50
+ elsewhere. Runs on ZeroGPU (`zero-a10g`).
app.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JEV-9B Decision API — one clean JSON call over System 1.
2
+
3
+ Inference core (`jev_core.py`) is the model authors' documented `transformers` + `peft`
4
+ path, reused verbatim from the official demo Space (Apache-2.0). This file only wraps it
5
+ in a single machine-friendly Gradio endpoint (`/decide`) that returns JSON, instead of the
6
+ upstream multi-question HTML UI, so a local agent can call it headlessly.
7
+
8
+ ZeroGPU rules honoured here:
9
+ * `import spaces` MUST precede torch / transformers / any CUDA-touching import
10
+ (jev_core imports torch, so `spaces` is imported before it).
11
+ * models are placed on `cuda` at MODULE level (inside jev_core, at import) — not
12
+ inside the `@spaces.GPU` function.
13
+ """
14
+
15
+ import os
16
+
17
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
18
+ os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False")
19
+
20
+ import spaces # noqa: E402 -- MUST precede torch / transformers / any CUDA-touching import
21
+
22
+ import json # noqa: E402
23
+
24
+ import gradio as gr # noqa: E402
25
+
26
+ import jev_core as jc # noqa: E402
27
+
28
+
29
+ def _clean_options(raw) -> list:
30
+ """Accept a comma/newline-separated string (or a list) -> list of option strings."""
31
+ if isinstance(raw, (list, tuple)):
32
+ items = [str(o).strip() for o in raw]
33
+ else:
34
+ items = [o.strip() for o in str(raw or "").replace("\n", ",").split(",")]
35
+ return [o for o in items if o]
36
+
37
+
38
+ @spaces.GPU(duration=45)
39
+ def decide(kind: str, state: str, question: str, options: str = "") -> str:
40
+ """Answer ONE typed System 1 decision; return calibrated probabilities as JSON.
41
+
42
+ kind: "noul" | "choice" | "score"
43
+ state: the text to reason about (truncated to 1024 tokens, head 60% / tail 40%)
44
+ question: the decision question
45
+ options: comma-separated options (choice only, 2-16); ignored for noul/score
46
+ """
47
+ kind = (kind or "").strip()
48
+ state = (state or "").strip()
49
+ question = (question or "").strip()
50
+
51
+ if kind not in jc.KINDS:
52
+ return json.dumps({"error": f"kind must be one of {list(jc.KINDS)}"})
53
+ if not state:
54
+ return json.dumps({"error": "state is required"})
55
+ if not question:
56
+ return json.dumps({"error": "question is required"})
57
+
58
+ opts = _clean_options(options)
59
+ if kind == "choice":
60
+ if not 2 <= len(opts) <= 16:
61
+ return json.dumps({"error": "choice needs 2-16 comma-separated options"})
62
+ if len(set(opts)) != len(opts):
63
+ return json.dumps({"error": "choice options must be unique"})
64
+ else:
65
+ opts = None
66
+
67
+ state_used, truncated, n_tok = jc.truncate_state(state)
68
+ dist = jc.decide(kind, state_used, question, opts)
69
+ best = max(dist, key=dist.get)
70
+ return json.dumps(
71
+ {
72
+ "kind": kind,
73
+ "distribution": dist,
74
+ "best": best,
75
+ "confidence": dist[best],
76
+ "state_tokens": n_tok,
77
+ "truncated": truncated,
78
+ }
79
+ )
80
+
81
+
82
+ with gr.Blocks(title="JEV-9B Decision API") as demo:
83
+ gr.Markdown(
84
+ "## JEV-9B System 1 — typed decisions in one prefill pass\n"
85
+ "`noul` (yes/no) · `choice` (2–16 options) · `score` (0–5). "
86
+ "Returns a calibrated probability distribution as JSON."
87
+ )
88
+ with gr.Row():
89
+ kind_in = gr.Dropdown(list(jc.KINDS), value="noul", label="kind")
90
+ state_in = gr.Textbox(label="state", lines=3)
91
+ question_in = gr.Textbox(label="question")
92
+ options_in = gr.Textbox(label="options (comma-separated; choice only)")
93
+ out_json = gr.Code(label="decision JSON", language="json")
94
+ btn = gr.Button("Decide", variant="primary")
95
+ btn.click(
96
+ decide,
97
+ inputs=[kind_in, state_in, question_in, options_in],
98
+ outputs=[out_json],
99
+ api_name="decide",
100
+ )
101
+
102
+ demo.launch()
jev_core.py ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JEV-9B inference core, ported 1:1 from the model authors' documented `transformers` +
2
+ `peft` path (https://huggingface.co/autotrust/JEV-9B -> "Other ways to run it").
3
+
4
+ The Blocks-of-Experts architecture:
5
+ * backbone = frozen Qwen3.5-9B text tower (bit-identical to the release)
6
+ * System 1 = LoRA r=16 (40.1M, shipped unmerged in `adapter/`) + a 24-slot fp32
7
+ decision head (`head.safetensors`) initialised from `lm_head` verbalizer rows
8
+ * System 2 = the original `lm_head`; used with the adapter off (pristine Qwen3.5-9B)
9
+
10
+ System 1 answers a typed question (`noul` yes/no, `choice` over 2-16 options, `score` on a
11
+ 0-5 scale) in ONE prefill pass and returns a calibrated probability distribution. The
12
+ decision prompt follows the authors' "bare-v1" template; the head is applied to the last
13
+ token's post-norm hidden state, divided by the per-kind calibration temperature.
14
+ """
15
+
16
+ import json
17
+ import os
18
+
19
+ import torch
20
+ from huggingface_hub import hf_hub_download
21
+ from peft import PeftModel
22
+ from safetensors.torch import load_file
23
+ from transformers import AutoModelForCausalLM, AutoTokenizer
24
+
25
+ REPO = "autotrust/JEV-9B"
26
+ LETTERS = "ABCDEFGHIJKLMNOP"
27
+ KINDS = ("noul", "choice", "score")
28
+ MAX_STATE_TOKENS = 1024 # the authors truncate long inputs to 1,024 tokens (head 60% / tail 40%)
29
+ DEVICE = os.environ.get("JEV_DEVICE", "cuda")
30
+ # Optional local-dir override (used only for offline verification of the wiring).
31
+ _LOCAL = os.environ.get("JEV_MODEL_PATH")
32
+
33
+
34
+ def _file(name: str) -> str:
35
+ """Resolve a model-repo file to a local path, falling back to the Hub."""
36
+ if _LOCAL:
37
+ p = os.path.join(_LOCAL, name)
38
+ if os.path.exists(p):
39
+ return p
40
+ return hf_hub_download(REPO, name)
41
+
42
+
43
+ _SRC = _LOCAL or REPO
44
+
45
+ # ---------------------------------------------------------------------------
46
+ # Model loading. On ZeroGPU the final `_peft.to(DEVICE)` is intercepted at
47
+ # startup: tensors are packed to disk and streamed into VRAM on the first
48
+ # @spaces.GPU entry, so the load cost is never paid inside a request.
49
+ #
50
+ # Everything is built on CPU first, then moved once:
51
+ # * peft's default would be `torch_device=infer_device()` -> "cuda", making
52
+ # safetensors read the adapter straight onto the GPU. That bypasses the
53
+ # packer and crashes the GPU-less main process ("No CUDA GPUs are
54
+ # available"). `torch_device="cpu"` keeps the read on the CPU.
55
+ # * one `.to(DEVICE)` at the end moves every tensor (backbone, LoRA, head)
56
+ # through the standard machinery.
57
+ # ---------------------------------------------------------------------------
58
+ _tok = AutoTokenizer.from_pretrained(_SRC)
59
+ _base = AutoModelForCausalLM.from_pretrained(_SRC, dtype=torch.bfloat16)
60
+
61
+ # Keep the adapter UNMERGED so System 2 (generation) runs with the adapter
62
+ # disabled -- `with _peft.disable_adapter():` -- i.e. pristine Qwen3.5-9B.
63
+ _peft = PeftModel.from_pretrained(_base, _SRC, subfolder="adapter", torch_device="cpu")
64
+
65
+ # System 1 head: fp32 linear, 4096 -> 24 slots.
66
+ _HEAD = torch.nn.Linear(4096, 24, dtype=torch.float32)
67
+ _head_sd = load_file(_file("head.safetensors"))
68
+ with torch.no_grad():
69
+ _HEAD.weight.copy_(_head_sd["proj.weight"])
70
+ _HEAD.bias.copy_(_head_sd["proj.bias"])
71
+ _HEAD.eval()
72
+
73
+ _JUDGE_CFG = json.load(open(_file("judge_config.json")))
74
+ _CALIB = json.load(open(_file("calibration.json")))["per_kind"]
75
+ SLOT_RANGES = {k: tuple(v) for k, v in _JUDGE_CFG["slots"]["ranges"].items()}
76
+ TEMPERATURES = {k: float(_CALIB[k]) for k in KINDS}
77
+
78
+ # Module handles: PeftModel -> LoraModel -> Qwen3_5ForCausalLM -> Qwen3_5TextModel.
79
+ _CAUSAL = _peft.base_model.model # the Qwen3_5ForCausalLM with LoRA modules inside
80
+ _BACKBONE = _CAUSAL.model # the text tower (LoRA active by default)
81
+
82
+ # Single move of the whole tree (backbone + LoRA + head) to the compute device.
83
+ _peft.to(DEVICE)
84
+ _HEAD.to(DEVICE)
85
+
86
+ # ---------------------------------------------------------------------------
87
+ # Prompt template -- exactly the authors' `bare-v1` decision format.
88
+ # ---------------------------------------------------------------------------
89
+
90
+
91
+ def build_decision_prompt(kind: str, state: str, question: str, options) -> tuple:
92
+ """Return (prompt_text, option_labels) in the authors' `bare-v1` format."""
93
+ if kind == "noul":
94
+ options = ["false", "true"]
95
+ elif kind == "score":
96
+ options = [str(i) for i in range(6)]
97
+ options = list(options)
98
+ if kind == "choice":
99
+ lines = [f"{LETTERS[i]}) {o}" for i, o in enumerate(options)]
100
+ else:
101
+ lines = options
102
+ prompt = (
103
+ f"[kind] {kind}\n[state] {state}\n[question] {question}\n[options]\n"
104
+ + "\n".join(lines)
105
+ + "\n[decision]:"
106
+ )
107
+ return prompt, options
108
+
109
+
110
+ def truncate_state(state: str) -> tuple:
111
+ """Head-60% / tail-40% truncation of the state to `MAX_STATE_TOKENS` tokens.
112
+
113
+ Returns (state_used, was_truncated, n_tokens).
114
+ """
115
+ ids = _tok(state, add_special_tokens=False)["input_ids"]
116
+ if len(ids) <= MAX_STATE_TOKENS:
117
+ return state, False, len(ids)
118
+ keep_head = int(MAX_STATE_TOKENS * 0.6)
119
+ keep_tail = MAX_STATE_TOKENS - keep_head
120
+ return _tok.decode(ids[:keep_head] + ids[-keep_tail:], skip_special_tokens=True), True, len(ids)
121
+
122
+
123
+ def decide(kind: str, state: str, question: str, options=None) -> dict:
124
+ """Run ONE System 1 typed decision; return the calibrated probability distribution.
125
+
126
+ This mirrors the model card's reference `decide()` byte for byte: one prefill pass
127
+ through the backbone with the LoRA active, the 24-slot head applied to the last
128
+ token's hidden state, per-kind temperature, softmax over the active slots.
129
+ """
130
+ if kind not in KINDS:
131
+ raise ValueError(f"unknown kind {kind!r}")
132
+ if kind == "choice":
133
+ if not isinstance(options, (list, tuple)) or not 2 <= len(options) <= 16:
134
+ raise ValueError("`choice` needs between 2 and 16 options")
135
+ if len(set(options)) != len(options):
136
+ raise ValueError("`choice` options must be unique")
137
+ prompt, labels = build_decision_prompt(kind, state, question, options)
138
+ device = _BACKBONE.embed_tokens.weight.device # resolve inside the GPU call
139
+ ids = _tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
140
+ with torch.no_grad():
141
+ h = _BACKBONE(**ids).last_hidden_state[0, -1].float()
142
+ z = _HEAD(h.to(_HEAD.weight.device)) # [24] fp32 logits
143
+ lo = SLOT_RANGES[kind][0]
144
+ z = z[lo : lo + len(labels)] / TEMPERATURES[kind]
145
+ p = torch.softmax(z, dim=0)
146
+ return {label: float(v) for label, v in zip(labels, p.tolist())}
147
+
148
+
149
+ def n_state_tokens(state: str) -> int:
150
+ return len(_tok(state, add_special_tokens=False)["input_ids"])
151
+
152
+
153
+ # ---------------------------------------------------------------------------
154
+ # System 2 -- ordinary generation from the pristine base model (adapter off).
155
+ # ---------------------------------------------------------------------------
156
+
157
+
158
+ def build_escalation_prompt(state: str, question: str, options) -> str:
159
+ """The model card's confidence-gated escalation prompt (System 1 -> System 2)."""
160
+ opts = [str(o) for o in options]
161
+ if opts == ["false", "true"]:
162
+ tail = "Answer `true` or `false` on the last line."
163
+ else:
164
+ tail = "Options: " + "; ".join(opts) + "\nThink it through, then give exactly one option on the last line."
165
+ return f"{state}\n\nQuestion: {question}\n{tail}"
166
+
167
+
168
+ def generate(state: str, question: str, options, max_new_tokens: int = 512,
169
+ enable_thinking: bool = True) -> tuple:
170
+ """System 2: generate a reasoned answer with the adapter OFF -- pristine Qwen3.5-9B.
171
+
172
+ Returns (text, completed): `completed` is False when generation hit the token
173
+ budget mid-sentence (the model is verbose in thinking mode), which the UI needs
174
+ to avoid presenting a cut-off trace as a finished answer.
175
+ """
176
+ prompt = build_escalation_prompt(state, question, options)
177
+ msgs = [{"role": "user", "content": prompt}]
178
+ enc = _tok.apply_chat_template(
179
+ msgs, add_generation_prompt=True, enable_thinking=enable_thinking,
180
+ return_tensors="pt", return_dict=True,
181
+ ).to(_BACKBONE.embed_tokens.weight.device)
182
+ n_prompt = enc["input_ids"].shape[1]
183
+ with torch.no_grad(), _peft.disable_adapter():
184
+ out = _CAUSAL.generate(**enc, max_new_tokens=int(max_new_tokens), do_sample=False)
185
+ text = _tok.decode(out[0, n_prompt:], skip_special_tokens=True)
186
+ completed = out.shape[1] - n_prompt < int(max_new_tokens) # stopped before the cap
187
+ return text.strip(), completed
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ transformers==5.16.1
2
+ peft==0.21.0
3
+ accelerate
4
+ safetensors