Text Generation
Transformers
Safetensors
English
llama
typed-decisions
decision-model
system-one
jev
jevbench
conversational
text-generation-inference
Instructions to use Kailune-AI/pjev-2b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Kailune-AI/pjev-2b with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Kailune-AI/pjev-2b") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("Kailune-AI/pjev-2b") model = AutoModelForCausalLM.from_pretrained("Kailune-AI/pjev-2b", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Kailune-AI/pjev-2b with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Kailune-AI/pjev-2b" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Kailune-AI/pjev-2b", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/Kailune-AI/pjev-2b
- SGLang
How to use Kailune-AI/pjev-2b with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Kailune-AI/pjev-2b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Kailune-AI/pjev-2b", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Kailune-AI/pjev-2b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Kailune-AI/pjev-2b", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use Kailune-AI/pjev-2b with Docker Model Runner:
docker model run hf.co/Kailune-AI/pjev-2b
Download letter_adapter.py from Kailune-AI/pjev-2b: direct link, hf CLI and curl.
- Browser
- Download file 21.2 kB
-
https://huggingface.co/Kailune-AI/pjev-2b/resolve/main/letter_adapter.py
- Command line
-
hf download hf://Kailune-AI/pjev-2b/letter_adapter.py
-
curl -L -o letter_adapter.py https://huggingface.co/Kailune-AI/pjev-2b/resolve/main/letter_adapter.py
21.2 kB
| """Decision readout from letter logits (one forward pass per decision), used in evaluation (the JevBench harness and | |
| the evaluation suites) and by the server. Format/prompt come from decision_core (same as in training). | |
| Modes: | |
| - local (transformers): model_path = model path/repo (+ optional LoRA adapter); | |
| - server: model_path = "sglang:<url>|<tokenizer>" (reads the letter logprobs from a running SGLang server); | |
| - vLLM: model_path = "vllm:<url>|<tokenizer>" (reads the letter logprobs from a vLLM server, see serve_vllm.sh); | |
| - verify: verify_budget > 0 → short reasoning (thinking) with a token budget, and only then the letter. | |
| """ | |
| from __future__ import annotations | |
| import contextlib | |
| import copy | |
| import json | |
| import math | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import Any, Optional | |
| import torch | |
| from decision_core import (LETTERS, PROMPT_VERSION, STYLE, SYSTEM, messages, options_of, # noqa: F401 | |
| render_user, state_text, temp_for, tournament) | |
| import os | |
| try: | |
| from jevbench.adapters.base import DecisionResult | |
| except ImportError: # used outside the harness (e.g. the release server) | |
| class DecisionResult: # type: ignore[no-redef] | |
| adapter: str | |
| ok: bool | |
| probs: Optional[dict] = None | |
| probs_source: str = "unknown" | |
| model: str = "" | |
| status: Optional[int] = None | |
| error: Optional[str] = None | |
| latency_s: float = 0.0 | |
| usage: dict = field(default_factory=dict) | |
| raw: Optional[Any] = None | |
| request_body: Optional[Any] = None | |
| label: Optional[str] = None | |
| class LetterAdapter: | |
| name = "ours_letter" | |
| cost_basis = "self_hosted_gpu" | |
| def __init__(self, model_path: str, device: str = "cuda", sym: bool = False, max_tokens: int = 12000, | |
| adapter_path: str | None = None, temp: float = 1.0, calib: str | None = None, | |
| verify_budget: int = 0, think_base: bool = False, readout: str | None = None, | |
| jepa_path: str | None = None, mix_w: float | None = None): | |
| self.model_path = model_path | |
| self.endpoint = model_path | |
| self.model = model_path | |
| self.device = device | |
| self.sym = sym | |
| self.max_tokens = max_tokens | |
| self.adapter_path = adapter_path | |
| self.temp = temp | |
| self.calib = json.load(open(calib)) if calib else None | |
| self.verify_budget = verify_budget | |
| # hybrid: the base model thinks (reasoning intact) and the trained adapter only reads the letter at the end | |
| self.think_base = bool(think_base and adapter_path and verify_budget > 0) | |
| # readout: "letter" (default), "energy" (JEPA: -cos between the predicted answer and the option spans) or "mix" | |
| import os | |
| self.readout = readout or os.environ.get("READOUT", "letter") | |
| self.jepa_path = jepa_path or os.environ.get("JEPA_HEAD") | |
| self.mix_w = mix_w if mix_w is not None else float(os.environ.get("MIX_W", "0.5")) | |
| self._jhead = None | |
| self._loaded = None | |
| # ---------------- loading ---------------- | |
| def _sglang(self): # "sglang:<url>|<tokenizer>" or "vllm:<url>|<tokenizer>" | |
| url, tok_path = self.model_path.split(":", 1)[1].split("|", 1) | |
| return url.rstrip("/"), tok_path | |
| def _is_vllm(self): | |
| return self.model_path.startswith("vllm:") | |
| def _vllm_letters(self, texts, n_max): | |
| """One /v1/completions call with all the prompts; allowed letters = A..(n_max); logprobs already normalized | |
| over the letters only (server run with --logprobs-mode processed_logprobs). Returns (list of letter_logprobs, | |
| n_tok).""" | |
| import urllib.request | |
| url, _ = self._sglang() | |
| want = self._loaded[2][:n_max] | |
| body = {"model": "decider", "prompt": texts, "max_tokens": 1, "temperature": 0.0, "logprobs": n_max, | |
| "allowed_token_ids": want, "return_tokens_as_token_ids": True} | |
| req = urllib.request.Request(f"{url}/v1/completions", data=json.dumps(body).encode(), | |
| headers={"Content-Type": "application/json"}) | |
| with urllib.request.urlopen(req, timeout=900) as r: | |
| d = json.loads(r.read()) | |
| out = [None] * len(texts) | |
| for ch in d["choices"]: | |
| top = ch["logprobs"]["top_logprobs"][0] | |
| lp = {int(k.split(":", 1)[1]) if k.startswith("token_id:") else k: v for k, v in top.items()} | |
| out[ch["index"]] = lp | |
| n_tok = d.get("usage", {}).get("prompt_tokens", 0) // max(1, len(texts)) | |
| return out, n_tok | |
| def _letter_ids(tok): | |
| ids = [] | |
| for L in LETTERS: | |
| t = tok.encode(L, add_special_tokens=False) | |
| assert len(t) == 1, f"letter {L} is not a single token: {t}" | |
| ids.append(t[0]) | |
| assert len(set(ids)) == len(ids) | |
| return ids | |
| def load(self): | |
| if self._loaded is not None: | |
| return self._loaded | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| if self.model_path.startswith(("sglang:", "vllm:")): | |
| tok = AutoTokenizer.from_pretrained(self._sglang()[1]) | |
| self._loaded = (None, tok, self._letter_ids(tok)) | |
| return self._loaded | |
| tok = AutoTokenizer.from_pretrained(self.model_path) | |
| model = AutoModelForCausalLM.from_pretrained(self.model_path, dtype=torch.bfloat16, | |
| device_map={"": self.device}) | |
| if self.adapter_path: | |
| from peft import PeftModel | |
| model = PeftModel.from_pretrained(model, self.adapter_path) | |
| if not self.think_base: | |
| model = model.merge_and_unload() | |
| model.eval() | |
| if self.readout != "letter": | |
| self._jhead = self._load_jhead(model) | |
| self._loaded = (model, tok, self._letter_ids(tok)) | |
| return self._loaded | |
| def _load_jhead(self, model): | |
| import torch.nn as nn | |
| ck = torch.load(self.jepa_path, map_location=self.device, weights_only=False) | |
| d, k = ck["d"], ck["dim"] | |
| class _H(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| mlp = lambda i, o: nn.Sequential(nn.Linear(i, 2 * k), nn.GELU(), nn.Linear(2 * k, o)) # noqa: E731 | |
| self.pred_ans, self.opt = mlp(d, k), nn.Linear(d, k) | |
| self.pred_view, self.pred_rat = mlp(d, d), mlp(d, d) | |
| h = _H() | |
| h.load_state_dict(ck["state"]) | |
| h.tau = ck["tau"] | |
| return h.to(self.device).float().eval() | |
| def prompt_ids(self, tok, user: str): # compat (default prompt) | |
| msgs = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}] | |
| text = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False) | |
| return tok(text, return_tensors="pt", add_special_tokens=False)["input_ids"] | |
| # ---------------- readout ---------------- | |
| def _probs(self, logits_letters, n_tok, opts, question): | |
| T = temp_for(self.calib, self.temp, n_tok, len(opts), question.get("type", "choice")) | |
| p = torch.softmax(logits_letters.float() / T, -1).tolist() | |
| return {lab: p[i] for i, (lab, _) in enumerate(opts)} | |
| def _post(self, body): | |
| import urllib.request | |
| url, _ = self._sglang() | |
| req = urllib.request.Request(f"{url}/generate", data=json.dumps(body).encode(), | |
| headers={"Content-Type": "application/json"}) | |
| with urllib.request.urlopen(req, timeout=900) as r: | |
| return json.loads(r.read()) | |
| def _letters_sglang(self, text, want): | |
| d = self._post({"text": text, "sampling_params": {"max_new_tokens": 1, "temperature": 0.0}, | |
| "return_logprob": True, "token_ids_logprob": want}) | |
| lp = {int(x[1]): float(x[0]) for x in d["meta_info"]["output_token_ids_logprobs"][0]} | |
| return torch.tensor([lp[i] for i in want]), int(d["meta_info"].get("prompt_tokens", 0)) | |
| def dist(self, state, question, opts): | |
| model, tok, let_ids = self.load() | |
| want = let_ids[:len(opts)] | |
| verify = self.verify_budget > 0 | |
| text = tok.apply_chat_template(messages(state, question, opts, verify=verify), tokenize=False, | |
| add_generation_prompt=True, enable_thinking=verify) | |
| if verify: # short reasoning with a budget; closes </think> if the budget runs out | |
| if model is None: | |
| d = self._post({"text": text, "sampling_params": {"max_new_tokens": self.verify_budget, | |
| "temperature": 0.0, "stop": ["</think>"]}}) | |
| thought = d["text"] | |
| else: | |
| ids = tok(text, return_tensors="pt", add_special_tokens=False)["input_ids"].to(self.device) | |
| ctx = model.disable_adapter() if self.think_base else contextlib.nullcontext() | |
| with ctx: | |
| g = model.generate(ids, max_new_tokens=self.verify_budget, do_sample=False, | |
| stop_strings=["</think>"], tokenizer=tok) | |
| thought = tok.decode(g[0, ids.shape[1]:], skip_special_tokens=False) | |
| if self.think_base: # readout in the training format (fast prompt) with the thought in the <think> block | |
| text = tok.apply_chat_template(messages(state, question, opts, verify=False), tokenize=False, | |
| add_generation_prompt=True, enable_thinking=True) | |
| text = text + thought.split("</think>")[0].rstrip() + "\n</think>\n\n" | |
| if model is None and self._is_vllm: | |
| (lp,), n_tok = self._vllm_letters([text], len(opts)) | |
| lg = torch.tensor([lp.get(i, -1e9) for i in want]) | |
| return self._probs(lg, n_tok, opts, question), n_tok | |
| if model is None: | |
| lg, n_tok = self._letters_sglang(text, want) | |
| return self._probs(lg, n_tok, opts, question), n_tok | |
| if self._jhead is not None and not verify: | |
| return self._dist_energy(model, tok, text, state, question, opts, want) | |
| ids = tok(text, return_tensors="pt", add_special_tokens=False)["input_ids"] | |
| if ids.shape[1] > self.max_tokens: | |
| raise ValueError(f"prompt with {ids.shape[1]} tokens > {self.max_tokens}") | |
| out = model(input_ids=ids.to(self.device), logits_to_keep=1) | |
| return self._probs(out.logits[0, -1, want], int(ids.shape[1]), opts, question), int(ids.shape[1]) | |
| def _dist_energy(self, model, tok, text, state, question, opts, want): | |
| """JEPA readout: energy between the predicted answer (last hidden state) and each option (its span in the | |
| context).""" | |
| from decision_core import option_spans, render_user | |
| enc = tok(text, add_special_tokens=False, return_offsets_mapping=True) | |
| ids = torch.tensor([enc["input_ids"]], device=self.device) | |
| if ids.shape[1] > self.max_tokens: | |
| raise ValueError(f"prompt with {ids.shape[1]} tokens > {self.max_tokens}") | |
| spans = option_spans(text, render_user(state, question, opts), opts, question, enc["offset_mapping"]) | |
| H = model.model(input_ids=ids).last_hidden_state[0] | |
| h = H[-1].float() | |
| p_let = self._probs(model.lm_head(H[-1:])[0, want], int(ids.shape[1]), opts, question) | |
| if not spans: | |
| return p_let, int(ids.shape[1]) | |
| pooled = torch.stack([H[a:b].float().mean(0) for a, b in spans]) | |
| z = torch.nn.functional.normalize(self._jhead.pred_ans(h), dim=-1) | |
| c = torch.nn.functional.normalize(self._jhead.opt(pooled), dim=-1) | |
| p_en = torch.softmax((c @ z) / self._jhead.tau, -1).tolist() | |
| p_en = {lab: p_en[i] for i, (lab, _) in enumerate(opts)} | |
| if self.readout == "energy": | |
| return p_en, int(ids.shape[1]) | |
| w = self.mix_w # product of experts: letters^(1-w) · energy^w | |
| mix = {k: math.exp((1 - w) * math.log(max(p_let[k], 1e-12)) + w * math.log(max(p_en[k], 1e-12))) for k in p_let} | |
| s = sum(mix.values()) | |
| return {k: v / s for k, v in mix.items()}, int(ids.shape[1]) | |
| def dist_many(self, state, items): | |
| """Parallel: several questions about the same state in a single GPU pass (batch, right-padding). | |
| items = [(question, opts), ...]. With vLLM or SGLang, all prompts go in one call and the server's prefix cache | |
| reuses the state. Falls back to the sequential path for the energy readout, verify mode, a single question or | |
| >26 options.""" | |
| model, tok, let_ids = self.load() | |
| if (self._jhead is not None or self.verify_budget > 0 or len(items) == 1 | |
| or any(len(o) > len(LETTERS) for _, o in items)): | |
| return [self.dist_any(state, q, o) for q, o in items] | |
| if model is None and self._is_vllm: # vLLM: one call; the server's prefix cache reuses the state | |
| texts = [tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) for q, o in items] | |
| lps, n_tok = self._vllm_letters(texts, max(len(o) for _, o in items)) | |
| return [(self._probs(torch.tensor([lp.get(i, -1e9) for i in let_ids[:len(o)]]), n_tok, o, q), n_tok) | |
| for (q, o), lp in zip(items, lps)] | |
| if model is None: # SGLang: one call with the list of prompts; the prefix cache processes the state once | |
| texts = [tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) for q, o in items] | |
| wants = [let_ids[:len(o)] for _, o in items] | |
| d = self._post({"text": texts, "sampling_params": [{"max_new_tokens": 1, "temperature": 0.0}] * len(texts), | |
| "return_logprob": [True] * len(texts), "token_ids_logprob": wants}) | |
| out = [] | |
| for (q, o), w, r in zip(items, wants, d): | |
| lp = {int(x[1]): float(x[0]) for x in r["meta_info"]["output_token_ids_logprobs"][0]} | |
| n_tok = int(r["meta_info"].get("prompt_tokens", 0)) | |
| out.append((self._probs(torch.tensor([lp[i] for i in w]), n_tok, o, q), n_tok)) | |
| return out | |
| seqs = [] | |
| for q, o in items: | |
| text = tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) | |
| seqs.append(tok(text, add_special_tokens=False)["input_ids"]) | |
| L = max(len(x) for x in seqs) | |
| if L > self.max_tokens: | |
| raise ValueError(f"prompt with {L} tokens > {self.max_tokens}") | |
| pad = tok.pad_token_id if tok.pad_token_id is not None else tok.eos_token_id | |
| ids = torch.full((len(seqs), L), pad, dtype=torch.long) | |
| att = torch.zeros((len(seqs), L), dtype=torch.long) | |
| for i, x in enumerate(seqs): | |
| ids[i, :len(x)] = torch.tensor(x) | |
| att[i, :len(x)] = 1 | |
| H = model.model(input_ids=ids.to(self.device), attention_mask=att.to(self.device)).last_hidden_state | |
| last = torch.tensor([len(x) - 1 for x in seqs], device=self.device) | |
| logits = model.lm_head(H[torch.arange(len(seqs), device=self.device), last]) | |
| return [(self._probs(logits[i, let_ids[:len(o)]], len(seqs[i]), o, q), len(seqs[i])) | |
| for i, (q, o) in enumerate(items)] | |
| def _expand_cache(cache, n): | |
| """Replicates the cache (attention: keys/values; linear: conv/recurrent) for a batch of n questions. | |
| Accepts both transformers formats: a plain tensor (5.12) or a dict keyed by state_idx (5.17+).""" | |
| def rep(t): | |
| if isinstance(t, torch.Tensor) and t.dim() > 0 and t.shape[0] == 1: | |
| return t.repeat_interleave(n, dim=0) | |
| return t | |
| for layer in getattr(cache, "layers", []): | |
| for attr in ("keys", "values", "conv_states", "recurrent_states"): | |
| t = getattr(layer, attr, None) | |
| if isinstance(t, dict): | |
| for k in t: | |
| t[k] = rep(t[k]) | |
| elif isinstance(t, list): | |
| t[:] = [rep(x) for x in t] | |
| else: | |
| setattr(layer, attr, rep(t)) | |
| return cache | |
| def dist_many_cached(self, state, items, return_timing=False): | |
| """Parallel, à la Jev: the common prefix (system + state) goes through the GPU once; the cache is replicated | |
| and the suffixes of the N questions run together in one batch. Same result as the full prompt.""" | |
| import time as _t | |
| model, tok, let_ids = self.load() | |
| if model is None or self._jhead is not None or self.verify_budget > 0 or any(len(o) > len(LETTERS) for _, o in items): | |
| return self.dist_many(state, items) | |
| texts = [tok.apply_chat_template(messages(state, q, o), tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) for q, o in items] | |
| seqs = tok(texts, add_special_tokens=False)["input_ids"] # batched tokenization (Rust) | |
| if max(len(x) for x in seqs) > self.max_tokens: | |
| raise ValueError(f"prompt with {max(len(x) for x in seqs)} tokens > {self.max_tokens}") | |
| P = 0 # longest common prefix in tokens (tokenization identical to that of the full prompt) | |
| while all(len(x) > P + 1 for x in seqs) and len({x[P] for x in seqs}) == 1: | |
| P += 1 | |
| t0 = _t.perf_counter() | |
| pre = torch.tensor([seqs[0][:P]], device=self.device) | |
| base_cache = model.model(input_ids=pre, use_cache=True).past_key_values | |
| t1 = _t.perf_counter() | |
| pad = tok.pad_token_id if tok.pad_token_id is not None else tok.eos_token_id | |
| out = [] | |
| CH = int(__import__("os").environ.get("CACHE_CHUNK", "64")) # questions per batch (replicated-cache memory) | |
| for c0 in range(0, len(seqs), CH): | |
| part = seqs[c0:c0 + CH] | |
| n = len(part) | |
| cache = self._expand_cache(copy.deepcopy(base_cache), n) | |
| suf = [x[P:] for x in part] | |
| S = max(len(x) for x in suf) | |
| ids = torch.full((n, S), pad, dtype=torch.long) | |
| att = torch.zeros((n, P + S), dtype=torch.long) | |
| att[:, :P] = 1 | |
| for i, x in enumerate(suf): | |
| ids[i, :len(x)] = torch.tensor(x) | |
| att[i, P:P + len(x)] = 1 | |
| H = model.model(input_ids=ids.to(self.device), attention_mask=att.to(self.device), past_key_values=cache, | |
| use_cache=True).last_hidden_state | |
| last = torch.tensor([len(x) - 1 for x in suf], device=self.device) | |
| logits = model.lm_head(H[torch.arange(n, device=self.device), last]) | |
| for i in range(n): | |
| q, o = items[c0 + i] | |
| out.append((self._probs(logits[i, let_ids[:len(o)]], len(part[i]), o, q), len(part[i]))) | |
| del cache, H | |
| if return_timing: | |
| if torch.cuda.is_available() and str(self.device).startswith("cuda"): | |
| torch.cuda.synchronize() | |
| return out, {"prefix_tokens": P, "max_suffix_tokens": S, "prefix_ms": round((t1 - t0) * 1000, 1), | |
| "total_ms": round((_t.perf_counter() - t0) * 1000, 1)} | |
| return out | |
| def dist_any(self, state, question, opts, chunk=20, keep=2): | |
| return tournament(lambda o: self.dist(state, question, o), opts, chunk, keep) | |
| # ---------------- harness ---------------- | |
| def run(self, task) -> DecisionResult: | |
| res = DecisionResult(adapter=self.name, ok=False, probs_source="native", model=self.model) | |
| try: | |
| self.load() | |
| except Exception as e: # noqa: BLE001 | |
| res.error = f"load failed: {type(e).__name__}: {str(e)[:250]}" | |
| return res | |
| opts = options_of(task.question, list(task.labels)) | |
| t0 = time.perf_counter() | |
| try: | |
| probs, n_in = self.dist_any(task.state, task.question, opts) | |
| if self.sym and len(opts) > 1: | |
| probs2, _ = self.dist_any(task.state, task.question, list(reversed(opts))) | |
| probs = {k: 0.5 * (probs[k] + probs2[k]) for k in probs} | |
| except Exception as e: # noqa: BLE001 | |
| res.latency_s = time.perf_counter() - t0 | |
| res.error = f"{type(e).__name__}: {str(e)[:300]}" | |
| return res | |
| if torch.cuda.is_available() and not self.model_path.startswith("sglang:"): | |
| torch.cuda.synchronize() | |
| res.latency_s = time.perf_counter() - t0 | |
| res.probs = probs | |
| res.usage = {"input_tokens": n_in * (2 if self.sym else 1), "output_tokens": 0} | |
| res.raw = {"prompt_version": PROMPT_VERSION, "sym": self.sym, "verify_budget": self.verify_budget} | |
| res.ok = True | |
| return res | |
| def reserve_estimate(self, task) -> float: | |
| return 0.0 | |