Ankit Nayak commited on
Commit
a053b64
·
verified ·
1 Parent(s): 5d89016

Upload README.md with huggingface_hub

Browse files
Files changed (1) hide show
  1. README.md +18 -68
README.md CHANGED
@@ -63,78 +63,28 @@ By compressing the 1.54B parameter transformer into 4-bit precision via `bitsand
63
 
64
  ---
65
 
66
- ## 🚀 Quickstart: Direct 4-Bit Inference
67
-
68
- ```bash
69
- pip install torch transformers safetensors accelerate bitsandbytes
70
- ```
71
 
72
  ```python
73
- import os
74
- import torch
75
- import torch.nn as nn
76
- import torch.nn.functional as F
77
- from transformers import AutoTokenizer, AutoModelForCausalLM
78
- from safetensors.torch import load_file
79
- from huggingface_hub import hf_hub_download
80
-
81
- class DecisionHead(nn.Module):
82
- def __init__(self, hidden_size: int = 1536, intermediate_size: int = 512, max_choices: int = 10, dropout: float = 0.1):
83
- super().__init__()
84
- self.max_choices = max_choices
85
- self.norm = nn.LayerNorm(hidden_size)
86
- self.dense = nn.Linear(hidden_size, intermediate_size)
87
- self.act = nn.GELU()
88
- self.dropout = nn.Dropout(dropout)
89
- self.out_proj = nn.Linear(intermediate_size, max_choices)
90
- self.score_proj = nn.Linear(intermediate_size, 1)
91
-
92
- def forward(self, x: torch.Tensor, num_choices: torch.Tensor = None) -> torch.Tensor:
93
- h = self.dropout(self.act(self.dense(self.norm(x))))
94
- logits = self.out_proj(h)
95
- if num_choices is not None:
96
- mask = torch.arange(self.max_choices, device=logits.device).unsqueeze(0) >= num_choices.unsqueeze(1)
97
- logits = logits.masked_fill(mask, -1e4)
98
- return logits
99
-
100
- # 1. Load 4-Bit Model directly (Transformers handles bitsandbytes automatically)
101
- model_id = "devankit7873/QJev-1.5B-4bit"
102
- device = "cuda" if torch.cuda.is_available() else "cpu"
103
-
104
- tokenizer = AutoTokenizer.from_pretrained(model_id)
105
- backbone = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
106
-
107
- # 2. Attach Decision Head
108
- head = DecisionHead().to(device=device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32)
109
- head_path = hf_hub_download(repo_id=model_id, filename="decision_head.safetensors")
110
- head.load_state_dict(load_file(head_path))
111
- head.eval()
112
-
113
- # 3. Predict with Calibrated Probabilities
114
- context = "Production database primary replica CPU spiked to 100% and connection pool is exhausted."
115
- choices = [
116
- "Trigger immediate automated failover to standby replica",
117
- "Restart the primary database daemon immediately",
118
- "Double connection pool limits without scaling compute",
119
- "Silence alerts and monitor for next 15 minutes"
120
- ]
121
-
122
- opts_str = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
123
- prompt = (
124
- "<|im_start|>system\nYou are an expert decision engine. Evaluate the context and select the optimal decision.<|im_end|>\n"
125
- f"<|im_start|>user\nContext:\n{context}\n\nQuestion:\nSelect the most appropriate option:\n\nOptions:\n{opts_str}<|im_end|>\n"
126
- "<|im_start|>assistant\nDecision:"
127
  )
128
 
129
- inputs = tokenizer(prompt, return_tensors="pt").to(device)
130
- with torch.no_grad():
131
- transformer = getattr(backbone, "model", backbone)
132
- out = transformer(inputs["input_ids"], attention_mask=inputs["attention_mask"])
133
- logits = head(out.last_hidden_state[:, -1, :], num_choices=torch.tensor([len(choices)], device=device))
134
- probs = F.softmax(logits[0, :len(choices)], dim=-1).cpu().tolist()
135
-
136
- for choice, prob in zip(choices, probs):
137
- print(f"{choice}: {prob * 100:.2f}%")
138
  ```
139
 
140
  ---
 
63
 
64
  ---
65
 
66
+ ## 🚀 Quickstart: Super Friendly 3-Line Decision
 
 
 
 
67
 
68
  ```python
69
+ from qjev import QJev
70
+
71
+ # 1. Load the 4-bit model in 1 line (only 1.07 GB VRAM):
72
+ model = QJev.from_pretrained("devankit7873/QJev-1.5B-4bit")
73
+
74
+ # 2. Decide in 35 ms!
75
+ result = model.decide(
76
+ context="Production database primary replica CPU spiked to 100% and connection pool is exhausted.",
77
+ choices=[
78
+ "Trigger immediate automated failover to standby replica",
79
+ "Restart the primary database daemon immediately",
80
+ "Double connection pool limits without scaling compute",
81
+ "Silence alerts and monitor for next 15 minutes"
82
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
83
  )
84
 
85
+ print(f"👉 Recommended: {result['best_choice']} ({result['confidence']}%)")
86
+ for choice, prob in result['probabilities'].items():
87
+ print(f" [{prob:>5.2f}%] {choice}")
 
 
 
 
 
 
88
  ```
89
 
90
  ---