Instructions to use michaelbzhu/test-7.6B-base with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use michaelbzhu/test-7.6B-base with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="michaelbzhu/test-7.6B-base", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("michaelbzhu/test-7.6B-base", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use michaelbzhu/test-7.6B-base with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "michaelbzhu/test-7.6B-base" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "michaelbzhu/test-7.6B-base", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/michaelbzhu/test-7.6B-base
- SGLang
How to use michaelbzhu/test-7.6B-base with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "michaelbzhu/test-7.6B-base" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "michaelbzhu/test-7.6B-base", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "michaelbzhu/test-7.6B-base" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "michaelbzhu/test-7.6B-base", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use michaelbzhu/test-7.6B-base with Docker Model Runner:
docker model run hf.co/michaelbzhu/test-7.6B-base
Download modeling.py from michaelbzhu/test-7.6B-base: direct link, hf CLI and curl.
- Browser
- Download file 5.63 kB
-
https://huggingface.co/michaelbzhu/test-7.6B-base/resolve/main/modeling.py
- Command line
-
hf download hf://michaelbzhu/test-7.6B-base/modeling.py
-
curl -L -o modeling.py https://huggingface.co/michaelbzhu/test-7.6B-base/resolve/main/modeling.py
5.63 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.nn.attention import sdpa_kernel, SDPBackend | |
| from transformers import PreTrainedModel | |
| from .configuration import MBZTestConfig | |
| from transformers.modeling_outputs import CausalLMOutput | |
| class RotaryPositionalEncoding(nn.Module): | |
| """ | |
| Rotary Position Embeddings (RoPE) - efficient implementation | |
| """ | |
| def __init__(self, d_head, max_seq_len=8192, base=10000.0): | |
| super().__init__() | |
| self.d_head = d_head | |
| self.max_seq_len = max_seq_len | |
| self.base = base | |
| # Precompute inverse frequencies | |
| inv_freq = 1.0 / (base ** (torch.arange(0, d_head, 2).float() / d_head)) | |
| self.register_buffer('inv_freq', inv_freq, persistent=False) | |
| # Precompute cos and sin for maximum sequence length | |
| self._precompute_freqs(max_seq_len) | |
| def _precompute_freqs(self, seq_len): | |
| """Precompute cos and sin values for positions""" | |
| t = torch.arange(seq_len, dtype=self.inv_freq.dtype, device=self.inv_freq.device) | |
| freqs = torch.outer(t, self.inv_freq) # (seq_len, d_head/2) | |
| # Create cos and sin embeddings | |
| freqs_cos = torch.cos(freqs) | |
| freqs_sin = torch.sin(freqs) | |
| # Interleave to match the dimension (seq_len, d_head) | |
| self.register_buffer('freqs_cos', freqs_cos.repeat_interleave(2, dim=-1), persistent=False) | |
| self.register_buffer('freqs_sin', freqs_sin.repeat_interleave(2, dim=-1), persistent=False) | |
| def rotate_half(self, x): | |
| """Rotate half the hidden dims of the input""" | |
| x1 = x[..., ::2] | |
| x2 = x[..., 1::2] | |
| return torch.stack([-x2, x1], dim=-1).flatten(-2) | |
| def forward(self, q, k, start_pos=0): | |
| """ | |
| Apply rotary embeddings to query and key tensors | |
| Args: | |
| q: (batch_size, n_heads, seq_len, d_head) | |
| k: (batch_size, n_heads, seq_len, d_head) | |
| start_pos: starting position for caching scenarios | |
| Returns: | |
| q_rot, k_rot with rotary embeddings applied | |
| """ | |
| seq_len = q.shape[2] | |
| # Get the precomputed frequencies for this sequence length | |
| freqs_cos = self.freqs_cos[start_pos:start_pos + seq_len] | |
| freqs_sin = self.freqs_sin[start_pos:start_pos + seq_len] | |
| # Apply rotary embeddings | |
| q_rot = q * freqs_cos + self.rotate_half(q) * freqs_sin | |
| k_rot = k * freqs_cos + self.rotate_half(k) * freqs_sin | |
| return q_rot, k_rot | |
| class Attention(nn.Module): | |
| def __init__(self, d_model, n_heads, d_head): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.n_heads = n_heads | |
| self.d_head = d_head | |
| self.Wq = nn.Linear(d_model, n_heads * d_head, bias=False) | |
| self.Wk = nn.Linear(d_model, n_heads * d_head, bias=False) | |
| self.Wv = nn.Linear(d_model, n_heads * d_head, bias=False) | |
| self.Wo = nn.Linear(n_heads * d_head, d_model, bias=False) | |
| # Initialize RoPE | |
| self.rope = RotaryPositionalEncoding(d_head) | |
| def forward(self, x): | |
| # x is shape batch_size, seq_len, d_model | |
| batch_size, seq_len, d_model = x.shape | |
| q = self.Wq(x) # q is shape batch_size, seq_len, n_heads * d_head | |
| k = self.Wk(x) | |
| v = self.Wv(x) | |
| # reshape to batch_size, n_heads, seq_len, d_head | |
| q = q.reshape(batch_size, seq_len, self.n_heads, self.d_head).transpose(1,2) | |
| k = k.reshape(batch_size, seq_len, self.n_heads, self.d_head).transpose(1,2) | |
| v = v.reshape(batch_size, seq_len, self.n_heads, self.d_head).transpose(1,2) | |
| q, k = self.rope(q, k) | |
| with sdpa_kernel(SDPBackend.FLASH_ATTENTION): # ensure use flash attention | |
| a = F.scaled_dot_product_attention(q, k, v, attn_mask=None, is_causal=True)# a is (batch_size, n_heads, seq_len, d_head) | |
| a = a.transpose(1,2) # change a to (batch_size, seq_len, n_heads, d_head) | |
| a = a.reshape(batch_size, seq_len, self.n_heads * self.d_head) | |
| out = self.Wo(a) # out is (batch_size, seq_len, d_model) | |
| return out | |
| class TransformerBlock(nn.Module): | |
| def __init__(self, d_model, n_heads, d_head): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.n_heads = n_heads | |
| self.d_head = d_head | |
| self.attn = Attention(d_model, n_heads, d_head) | |
| self.mlp = nn.Sequential(nn.Linear(d_model, 4*d_model), nn.ReLU(), nn.Linear(4*d_model, d_model)) | |
| self.norm1 = nn.RMSNorm(d_model) | |
| self.norm2 = nn.RMSNorm(d_model) | |
| def forward(self, x): | |
| x = self.attn(self.norm1(x)) + x | |
| x = self.mlp(self.norm2(x)) + x | |
| return x | |
| class MBZTestModelForCausalLM(PreTrainedModel): | |
| config_class = MBZTestConfig | |
| def __init__(self, config): | |
| super().__init__(config) | |
| d_model = config.d_model | |
| n_heads = config.n_heads | |
| d_head = config.d_head | |
| n_vocab = config.n_vocab | |
| n_layers = config.n_layers | |
| self.d_model = d_model | |
| self.n_heads = n_heads | |
| self.d_head = d_head | |
| self.n_vocab = n_vocab | |
| self.embed = nn.Embedding(n_vocab, d_model) | |
| self.blocks = nn.ModuleList([TransformerBlock(d_model, n_heads, d_head) for _ in range(n_layers)]) | |
| self.norm = nn.RMSNorm(d_model) | |
| self.out_head = nn.Linear(d_model, n_vocab) | |
| def forward(self, x): | |
| with torch.autocast('cuda', dtype=torch.bfloat16): | |
| x = self.embed(x) | |
| for block in self.blocks: | |
| x = block(x) | |
| x = self.out_head(self.norm(x)) | |
| return CausalLMOutput(logits=x) | |