Instructions to use Jonnester/Purrence-3M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Jonnester/Purrence-3M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Jonnester/Purrence-3M", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("Jonnester/Purrence-3M", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Jonnester/Purrence-3M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Jonnester/Purrence-3M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jonnester/Purrence-3M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/Jonnester/Purrence-3M
- SGLang
How to use Jonnester/Purrence-3M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Jonnester/Purrence-3M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jonnester/Purrence-3M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Jonnester/Purrence-3M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jonnester/Purrence-3M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use Jonnester/Purrence-3M with Docker Model Runner:
docker model run hf.co/Jonnester/Purrence-3M
Download purrence_runtime/scorer.py from Jonnester/Purrence-3M: direct link, hf CLI and curl.
- Browser
- Download file 5.79 kB
-
https://huggingface.co/Jonnester/Purrence-3M/resolve/main/purrence_runtime/scorer.py
- Command line
-
hf download hf://Jonnester/Purrence-3M/purrence_runtime/scorer.py
-
curl -L -o scorer.py https://huggingface.co/Jonnester/Purrence-3M/resolve/main/purrence_runtime/scorer.py
5.79 kB
| from contextlib import contextmanager | |
| import torch | |
| import torch.nn.functional as F | |
| from .tokenizer import Sub3MTokenizer | |
| from pathlib import Path | |
| TOKENIZER = Path(__file__).resolve().parents[1] / "tokenizer" | |
| def cross_entropy_sum(logits, targets): | |
| return F.cross_entropy(logits.float(), targets, reduction="sum") | |
| def fp32_math(device): | |
| """Full FP32 inference, including attention, without leaking backend flags.""" | |
| from torch.nn.attention import SDPBackend, sdpa_kernel | |
| previous_precision = torch.get_float32_matmul_precision() | |
| previous_tf32 = torch.backends.cuda.matmul.allow_tf32 | |
| previous_cudnn_tf32 = torch.backends.cudnn.allow_tf32 | |
| try: | |
| torch.set_float32_matmul_precision("highest") | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| torch.backends.cudnn.allow_tf32 = False | |
| with torch.autocast(device.type, enabled=False), sdpa_kernel(SDPBackend.MATH): | |
| yield | |
| finally: | |
| torch.set_float32_matmul_precision(previous_precision) | |
| torch.backends.cuda.matmul.allow_tf32 = previous_tf32 | |
| torch.backends.cudnn.allow_tf32 = previous_cudnn_tf32 | |
| class Scorer: | |
| def __init__(self, model, tokenizer=None, batch_size=16): | |
| if batch_size < 1: | |
| raise ValueError("Batch size must be positive") | |
| self.model = model.eval() | |
| self.tokenizer = tokenizer or Sub3MTokenizer(TOKENIZER) | |
| self.batch_size = batch_size | |
| self.device = next(model.parameters()).device | |
| if any(p.is_floating_point() and p.dtype != torch.float32 | |
| for p in model.parameters()): | |
| raise ValueError("This scorer requires FP32 inference weights") | |
| def inference_context(self): | |
| return fp32_math(self.device) | |
| def score(self, requests, max_length=None): | |
| limit = max_length or self.model.config.max_seq_len | |
| if not 1 <= limit <= self.model.config.max_seq_len: | |
| raise ValueError("Scoring context is outside the model's context limit") | |
| results = [] | |
| for begin in range(0, len(requests), self.batch_size): | |
| inputs, targets, lengths, counts = [], [], [], [] | |
| for context, continuation in requests[begin:begin + self.batch_size]: | |
| if not continuation or len(continuation) > limit: | |
| raise ValueError("Continuation must contain between one and max_length tokens") | |
| context = context or [self.tokenizer.boundary_id] | |
| sequence = (list(context) + list(continuation))[-(limit + 1):] | |
| inputs.extend(sequence[:-1]) | |
| targets.append(continuation) | |
| lengths.append(len(sequence) - 1) | |
| counts.append(len(continuation)) | |
| ids = torch.tensor(inputs, dtype=torch.long, device=self.device) | |
| cu = F.pad(torch.tensor(lengths, device=self.device).cumsum(0), (1, 0)).to(torch.int32) | |
| positions = torch.cat([torch.arange(length, device=self.device) for length in lengths]) | |
| with self.inference_context(): | |
| logits = self.model(ids, cu, positions) | |
| offset = 0 | |
| for length, count, target in zip(lengths, counts, targets): | |
| selected = logits[offset + length - count:offset + length].float() | |
| target = torch.tensor(target, device=self.device) | |
| value = -cross_entropy_sum(selected, target) | |
| greedy = selected.argmax(-1).eq(target).all() | |
| results.append((float(value), bool(greedy))) | |
| offset += length | |
| return results | |
| def rolling(self, text): | |
| tokens = self.tokenizer.encode(text) | |
| limit = self.model.config.max_seq_len | |
| requests = [] | |
| for start in range(0, len(tokens), limit): | |
| continuation = tokens[start:start + limit] | |
| if not start: | |
| context = [self.tokenizer.boundary_id] | |
| elif len(continuation) < limit: | |
| # LM-Eval's final partial window keeps enough preceding tokens | |
| # to use the full model context; full-sized windows keep one. | |
| context_size = limit - len(continuation) + 1 | |
| context = tokens[start - context_size:start] | |
| else: | |
| context = [tokens[start - 1]] | |
| requests.append((context, continuation)) | |
| return sum(score for score, _ in self.score(requests)) | |
| def generate(self, prompt, max_new_tokens=128, temperature=0.0, until=(), seed=17): | |
| if max_new_tokens < 0 or temperature < 0: | |
| raise ValueError("Generation length and temperature must be nonnegative") | |
| if isinstance(until, str): | |
| until = [until] | |
| if any(not stop for stop in until): | |
| raise ValueError("Stop strings must be nonempty") | |
| prefix = self.tokenizer.encode(prompt) or [self.tokenizer.boundary_id] | |
| generated = [] | |
| rng = torch.Generator(device=self.device).manual_seed(seed) | |
| for _ in range(max_new_tokens): | |
| ids = torch.tensor((prefix + generated)[-self.model.config.max_seq_len:], device=self.device) | |
| with self.inference_context(): | |
| logits = self.model(ids)[-1].float() | |
| token = int(logits.argmax()) if temperature == 0 else int(torch.multinomial( | |
| (logits / temperature).softmax(-1), 1, generator=rng)) | |
| if token == self.tokenizer.boundary_id: | |
| break | |
| generated.append(token) | |
| text = self.tokenizer.decode(generated) | |
| endings = [text.index(stop) for stop in until if stop in text] | |
| if endings: | |
| return text[:min(endings)] | |
| return self.tokenizer.decode(generated) | |