Text Classification
Transformers
Safetensors
PyTorch
qwen2
text-generation
bitsandbytes
4-bit precision
quantized
qwen
decision-making
system-one
reasoning
jev
jevbench
joint-expected-value
classification
agent
fast-inference
text-embeddings-inference
Instructions to use AnkitNayak-dev/QJev-1.5B-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use AnkitNayak-dev/QJev-1.5B-4bit with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="AnkitNayak-dev/QJev-1.5B-4bit")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("AnkitNayak-dev/QJev-1.5B-4bit") model = AutoModelForCausalLM.from_pretrained("AnkitNayak-dev/QJev-1.5B-4bit", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Ankit Nayak commited on
Upload README.md with huggingface_hub
Browse files
README.md
CHANGED
|
@@ -63,78 +63,28 @@ By compressing the 1.54B parameter transformer into 4-bit precision via `bitsand
|
|
| 63 |
|
| 64 |
---
|
| 65 |
|
| 66 |
-
## 🚀 Quickstart:
|
| 67 |
-
|
| 68 |
-
```bash
|
| 69 |
-
pip install torch transformers safetensors accelerate bitsandbytes
|
| 70 |
-
```
|
| 71 |
|
| 72 |
```python
|
| 73 |
-
import
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
self.act = nn.GELU()
|
| 88 |
-
self.dropout = nn.Dropout(dropout)
|
| 89 |
-
self.out_proj = nn.Linear(intermediate_size, max_choices)
|
| 90 |
-
self.score_proj = nn.Linear(intermediate_size, 1)
|
| 91 |
-
|
| 92 |
-
def forward(self, x: torch.Tensor, num_choices: torch.Tensor = None) -> torch.Tensor:
|
| 93 |
-
h = self.dropout(self.act(self.dense(self.norm(x))))
|
| 94 |
-
logits = self.out_proj(h)
|
| 95 |
-
if num_choices is not None:
|
| 96 |
-
mask = torch.arange(self.max_choices, device=logits.device).unsqueeze(0) >= num_choices.unsqueeze(1)
|
| 97 |
-
logits = logits.masked_fill(mask, -1e4)
|
| 98 |
-
return logits
|
| 99 |
-
|
| 100 |
-
# 1. Load 4-Bit Model directly (Transformers handles bitsandbytes automatically)
|
| 101 |
-
model_id = "devankit7873/QJev-1.5B-4bit"
|
| 102 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 103 |
-
|
| 104 |
-
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 105 |
-
backbone = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
|
| 106 |
-
|
| 107 |
-
# 2. Attach Decision Head
|
| 108 |
-
head = DecisionHead().to(device=device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32)
|
| 109 |
-
head_path = hf_hub_download(repo_id=model_id, filename="decision_head.safetensors")
|
| 110 |
-
head.load_state_dict(load_file(head_path))
|
| 111 |
-
head.eval()
|
| 112 |
-
|
| 113 |
-
# 3. Predict with Calibrated Probabilities
|
| 114 |
-
context = "Production database primary replica CPU spiked to 100% and connection pool is exhausted."
|
| 115 |
-
choices = [
|
| 116 |
-
"Trigger immediate automated failover to standby replica",
|
| 117 |
-
"Restart the primary database daemon immediately",
|
| 118 |
-
"Double connection pool limits without scaling compute",
|
| 119 |
-
"Silence alerts and monitor for next 15 minutes"
|
| 120 |
-
]
|
| 121 |
-
|
| 122 |
-
opts_str = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
|
| 123 |
-
prompt = (
|
| 124 |
-
"<|im_start|>system\nYou are an expert decision engine. Evaluate the context and select the optimal decision.<|im_end|>\n"
|
| 125 |
-
f"<|im_start|>user\nContext:\n{context}\n\nQuestion:\nSelect the most appropriate option:\n\nOptions:\n{opts_str}<|im_end|>\n"
|
| 126 |
-
"<|im_start|>assistant\nDecision:"
|
| 127 |
)
|
| 128 |
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
out = transformer(inputs["input_ids"], attention_mask=inputs["attention_mask"])
|
| 133 |
-
logits = head(out.last_hidden_state[:, -1, :], num_choices=torch.tensor([len(choices)], device=device))
|
| 134 |
-
probs = F.softmax(logits[0, :len(choices)], dim=-1).cpu().tolist()
|
| 135 |
-
|
| 136 |
-
for choice, prob in zip(choices, probs):
|
| 137 |
-
print(f"{choice}: {prob * 100:.2f}%")
|
| 138 |
```
|
| 139 |
|
| 140 |
---
|
|
|
|
| 63 |
|
| 64 |
---
|
| 65 |
|
| 66 |
+
## 🚀 Quickstart: Super Friendly 3-Line Decision
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
```python
|
| 69 |
+
from qjev import QJev
|
| 70 |
+
|
| 71 |
+
# 1. Load the 4-bit model in 1 line (only 1.07 GB VRAM):
|
| 72 |
+
model = QJev.from_pretrained("devankit7873/QJev-1.5B-4bit")
|
| 73 |
+
|
| 74 |
+
# 2. Decide in 35 ms!
|
| 75 |
+
result = model.decide(
|
| 76 |
+
context="Production database primary replica CPU spiked to 100% and connection pool is exhausted.",
|
| 77 |
+
choices=[
|
| 78 |
+
"Trigger immediate automated failover to standby replica",
|
| 79 |
+
"Restart the primary database daemon immediately",
|
| 80 |
+
"Double connection pool limits without scaling compute",
|
| 81 |
+
"Silence alerts and monitor for next 15 minutes"
|
| 82 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
)
|
| 84 |
|
| 85 |
+
print(f"👉 Recommended: {result['best_choice']} ({result['confidence']}%)")
|
| 86 |
+
for choice, prob in result['probabilities'].items():
|
| 87 |
+
print(f" [{prob:>5.2f}%] {choice}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
```
|
| 89 |
|
| 90 |
---
|