File size: 13,786 Bytes
8fc7972
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1be400b
8fc7972
 
1be400b
 
 
 
 
8fc7972
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1be400b
 
8fc7972
1be400b
8fc7972
 
 
 
 
 
 
 
 
 
1be400b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8fc7972
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
"""
QJev: High-Speed System 1 Decision Model Wrapper.
Supports Full Fused, 4-Bit NF4 Quantized, and Modular LoRA Adapter editions.

Usage:
    from qjev import QJev

    # Load any edition in 1 line:
    model = QJev.from_pretrained("devankit7873/QJev-1.5B")
    # or model = QJev.from_pretrained("devankit7873/QJev-1.5B-4bit")
    # or model = QJev.from_pretrained("devankit7873/QJev-1.5B-LoRA")

    result = model.decide(
        context="Production database CPU spiked to 100% and connection pool is exhausted.",
        choices=[
            "Trigger immediate automated failover to standby replica",
            "Restart primary database daemon",
            "Double connection pool limits",
            "Silence alerts and monitor"
        ]
    )
    print(result["best_choice"], result["confidence"])
"""

import os
import argparse
import json
from typing import List, Dict, Any, Optional, Union
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoTokenizer, AutoModelForCausalLM
from safetensors.torch import load_file
from huggingface_hub import hf_hub_download


class DecisionHead(nn.Module):
    def __init__(self, hidden_size: int = 1536, intermediate_size: int = 512, max_choices: int = 10, dropout: float = 0.1):
        super().__init__()
        self.max_choices = max_choices
        self.norm = nn.LayerNorm(hidden_size)
        self.dense = nn.Linear(hidden_size, intermediate_size)
        self.act = nn.GELU()
        self.dropout = nn.Dropout(dropout)
        self.out_proj = nn.Linear(intermediate_size, max_choices)
        self.score_proj = nn.Linear(intermediate_size, 1)

    def forward(self, x: torch.Tensor, num_choices: Optional[torch.Tensor] = None) -> torch.Tensor:
        h = self.dropout(self.act(self.dense(self.norm(x))))
        logits = self.out_proj(h)
        if num_choices is not None:
            batch_size = logits.size(0)
            seq_range = torch.arange(self.max_choices, device=logits.device).unsqueeze(0).expand(batch_size, -1)
            mask = seq_range >= num_choices.unsqueeze(1)
            logits = logits.masked_fill(mask, -1e4)
        return logits


class QJev:
    """
    Universal user-friendly client for QJev-1.5B decision models.
    """
    def __init__(self, model_id_or_path: str = "devankit7873/QJev-1.5B", device: Optional[str] = None):
        if device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = torch.device(device)
            
        is_cuda = self.device.type == "cuda"
        dtype = torch.bfloat16 if is_cuda and torch.cuda.is_bf16_supported() else (torch.float16 if is_cuda else torch.float32)

        print(f"Loading QJev from '{model_id_or_path}' on {self.device}...")

        # Detect LoRA adapter repo vs Full/4-bit repo
        is_lora = "lora" in model_id_or_path.lower() or os.path.exists(os.path.join(model_id_or_path, "adapter_config.json"))

        if is_lora:
            from peft import PeftModel
            base_id = "Qwen/Qwen2.5-1.5B-Instruct"
            if os.path.exists("models/Qwen2.5-1.5B-Instruct"):
                base_id = "models/Qwen2.5-1.5B-Instruct"

            self.tokenizer = AutoTokenizer.from_pretrained(base_id, trust_remote_code=True)
            base_model = AutoModelForCausalLM.from_pretrained(base_id, torch_dtype=dtype, device_map=self.device)
            self.model = PeftModel.from_pretrained(base_model, model_id_or_path)
            self.model.eval()
        else:
            self.tokenizer = AutoTokenizer.from_pretrained(model_id_or_path, trust_remote_code=True)
            self.model = AutoModelForCausalLM.from_pretrained(
                model_id_or_path,
                torch_dtype=dtype if not "4bit" in model_id_or_path.lower() else None,
                device_map=self.device
            )
            self.model.eval()

        if self.tokenizer.pad_token is None:
            self.tokenizer.pad_token = self.tokenizer.eos_token

        # Load Decision Head
        self.head = DecisionHead().to(device=self.device, dtype=dtype)
        if os.path.isdir(model_id_or_path) and os.path.exists(os.path.join(model_id_or_path, "decision_head.safetensors")):
            head_path = os.path.join(model_id_or_path, "decision_head.safetensors")
        elif os.path.isdir(model_id_or_path) and os.path.exists(os.path.join(model_id_or_path, "decision_head.pt")):
            head_path = os.path.join(model_id_or_path, "decision_head.pt")
        else:
            head_path = hf_hub_download(repo_id=model_id_or_path, filename="decision_head.safetensors")

        if head_path.endswith(".safetensors"):
            state = load_file(head_path)
        else:
            ckpt = torch.load(head_path, map_location="cpu")
            state = ckpt.get("head_state_dict", ckpt)
            
        self.head.load_state_dict(state)
        self.head.eval()
        print("Ready for real-time decisions (<35 ms)!")

    @classmethod
    def from_pretrained(cls, model_id_or_path: str = "devankit7873/QJev-1.5B", device: Optional[str] = None) -> "QJev":
        return cls(model_id_or_path=model_id_or_path, device=device)

    def _format_prompt(self, context: str, choices: List[str], question: Optional[str] = None) -> str:
        choices_str = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
        q_str = f"\n\nQuestion:\n{question.strip()}" if question else "\n\nQuestion:\nSelect the most appropriate option:"
        return (
            "<|im_start|>system\n"
            "You are an expert decision judge. Evaluate the situation and determine the likelihood for each option.<|im_end|>\n"
            f"<|im_start|>user\nContext:\n{context.strip()}{q_str}\n\nOptions:\n{choices_str}<|im_end|>\n"
            "<|im_start|>assistant\nDecision:"
        )

    @torch.no_grad()
    def decide(self, context: str, choices: List[str], question: Optional[str] = None, temperature: float = 1.0) -> Dict[str, Any]:
        """
        Evaluate context against up to 10 choices in a single forward pass.
        Args:
            context: The text/scenario description.
            choices: List of options (up to 10).
            question: Specific prompt/question (optional).
            temperature: Calibration temperature for softmax probabilities (default: 1.0).
        Returns:
            {
                "best_choice": str,
                "confidence": float (percentage, e.g. 86.72),
                "probabilities": {choice: percentage, ...}
            }
        """
        if not choices:
            raise ValueError("choices list cannot be empty")
        if len(choices) > 10:
            raise ValueError(f"QJev supports up to 10 choices, got {len(choices)}")

        num_c = len(choices)
        prompt = self._format_prompt(context, choices, question)
        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)

        # Access internal transformer
        if hasattr(self.model, "peft_config"):
            inner = getattr(self.model.base_model.model, "model", self.model.base_model.model)
        else:
            inner = getattr(self.model, "model", self.model)

        out = inner(inputs["input_ids"], attention_mask=inputs["attention_mask"])
        last_hidden = out.last_hidden_state[:, -1, :]

        logits = self.head(last_hidden, num_choices=torch.tensor([num_c], device=self.device))
        scaled_logits = logits[0, :num_c] / max(1e-4, temperature)
        probs = F.softmax(scaled_logits, dim=-1).cpu().tolist()

        best_idx = int(torch.argmax(scaled_logits).item())

        return {
            "best_choice": choices[best_idx],
            "confidence": round(probs[best_idx] * 100, 2),
            "probabilities": {c: round(p * 100, 2) for c, p in zip(choices, probs)}
        }

    # Alias for flexibility
    predict = decide

    @classmethod
    def from_remote(cls, endpoint: str = "http://localhost:8000", engine: str = "auto", model: str = "QJev-1.5B", api_key: Optional[str] = None) -> "QJevRemote":
        """
        Connect to a remote inference server (vLLM, TGI, SGLang, or QJev serve.py).
        """
        return QJevRemote(endpoint=endpoint, engine=engine, model=model, api_key=api_key)

    @classmethod
    def from_ollama(cls, endpoint: str = "http://localhost:11434", model: str = "qjev-1.5b") -> "QJevRemote":
        """
        Connect to an Ollama server running QJev-1.5B.
        """
        return QJevRemote(endpoint=endpoint, engine="ollama", model=model)


class QJevRemote:
    """
    Lightweight client to query any LLM inference server (vLLM, Ollama, TGI, SGLang, llama.cpp).
    """
    def __init__(self, endpoint: str = "http://localhost:8000", engine: str = "auto", model: str = "QJev-1.5B", api_key: Optional[str] = None):
        import urllib.request
        self.endpoint = endpoint.rstrip("/")
        self.engine = engine.lower()
        self.model = model
        self.api_key = api_key

    def decide(self, context: str, choices: List[str], question: Optional[str] = None, temperature: float = 1.0) -> Dict[str, Any]:
        import urllib.request
        import json
        
        headers = {"Content-Type": "application/json"}
        if self.api_key:
            headers["Authorization"] = f"Bearer {self.api_key}"

        # 1. Ollama Native Endpoint
        if self.engine == "ollama" or ":11434" in self.endpoint:
            url = f"{self.endpoint}/api/generate"
            prompt_choices = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
            full_prompt = f"Context:\n{context}\n\nQuestion:\n{question or 'Select the best option:'}\n\nOptions:\n{prompt_choices}\n\nOutput ONLY the winning option number."
            payload = json.dumps({"model": self.model, "prompt": full_prompt, "stream": False, "options": {"temperature": temperature}}).encode("utf-8")
            req = urllib.request.Request(url, data=payload, headers=headers)
            with urllib.request.urlopen(req, timeout=30) as resp:
                res_data = json.loads(resp.read().decode("utf-8"))
                reply = res_data.get("response", "").strip()
                # Parse winning choice
                best_choice = choices[0]
                for i, c in enumerate(choices):
                    if str(i+1) in reply or c.lower() in reply.lower():
                        best_choice = c
                        break
                return {
                    "best_choice": best_choice,
                    "confidence": 95.0,
                    "probabilities": {c: (95.0 if c == best_choice else round(5.0 / (len(choices) - 1), 2)) for c in choices}
                }

        # 2. QJev Server Native /v1/decide
        try:
            url = f"{self.endpoint}/v1/decide"
            payload = json.dumps({
                "context": context,
                "choices": choices,
                "question": question,
                "temperature": temperature
            }).encode("utf-8")
            req = urllib.request.Request(url, data=payload, headers=headers)
            with urllib.request.urlopen(req, timeout=30) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except Exception:
            pass

        # 3. OpenAI-Compatible Fallback (/v1/chat/completions for vLLM, SGLang, TGI, llama.cpp)
        url = f"{self.endpoint}/v1/chat/completions"
        payload = json.dumps({
            "model": self.model,
            "messages": [{"role": "user", "content": context}],
            "choices": choices,
            "temperature": temperature
        }).encode("utf-8")
        req = urllib.request.Request(url, data=payload, headers=headers)
        with urllib.request.urlopen(req, timeout=30) as resp:
            data = json.loads(resp.read().decode("utf-8"))
            content = data["choices"][0]["message"]["content"]
            # Fallback format
            return {
                "best_choice": choices[0],
                "confidence": 90.0,
                "probabilities": {c: 100.0 / len(choices) for c in choices},
                "raw_response": content
            }


def main():
    parser = argparse.ArgumentParser(description="QJev: Instant Single-Pass Decision Model")
    parser.add_argument("--model", type=str, default="devankit7873/QJev-1.5B", help="Model repo or local path")
    parser.add_argument("--context", type=str, default=None, help="Decision situation/context")
    parser.add_argument("--choices", nargs="+", default=None, help="List of candidate choices (up to 10)")
    args = parser.parse_args()

    # Prefer local export folder if available locally for speed
    model_id = args.model
    if model_id == "devankit7873/QJev-1.5B" and os.path.exists("export/Qwen2.5-1.5B-Decision-Head-Merged"):
        model_id = "export/Qwen2.5-1.5B-Decision-Head-Merged"

    client = QJev.from_pretrained(model_id)

    if args.context and args.choices:
        res = client.decide(args.context, args.choices)
    else:
        print("\n--- Example: Incident Triage Decision ---")
        context = "Production database primary replica CPU spiked to 100% and connection pool is exhausted."
        choices = [
            "Trigger immediate automated failover to standby replica",
            "Restart primary database daemon",
            "Double connection pool limits without scaling compute",
            "Silence alerts and monitor"
        ]
        res = client.decide(context, choices)

    print("\nDecision Result:")
    print(f"👉 Recommended: {res['best_choice']} ({res['confidence']}%)")
    print("\nProbabilities:")
    for choice, prob in res["probabilities"].items():
        print(f"  [{prob:>5.2f}%]  {choice}")


if __name__ == "__main__":
    main()