Text Generation
MLX
Safetensors
Italian
English
mistral
mlx-lm
8-bit precision
apple-silicon
italian
velvet
conversational
Instructions to use rbizzoni/Velvet2-3B-MLX-8bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use rbizzoni/Velvet2-3B-MLX-8bit with MLX:
# Make sure mlx-lm is installed # pip install --upgrade mlx-lm # Generate text with mlx-lm from mlx_lm import load, generate model, tokenizer = load("rbizzoni/Velvet2-3B-MLX-8bit") prompt = "Write a story about Einstein" messages = [{"role": "user", "content": prompt}] prompt = tokenizer.apply_chat_template( messages, add_generation_prompt=True ) text = generate(model, tokenizer, prompt=prompt, verbose=True) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use rbizzoni/Velvet2-3B-MLX-8bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "rbizzoni/Velvet2-3B-MLX-8bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "rbizzoni/Velvet2-3B-MLX-8bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- MLX LM
How to use rbizzoni/Velvet2-3B-MLX-8bit with MLX LM:
Generate or start a chat session
# Install MLX LM uv tool install mlx-lm # Interactive chat REPL mlx_lm.chat --model "rbizzoni/Velvet2-3B-MLX-8bit"
Run an OpenAI-compatible server
# Install MLX LM uv tool install mlx-lm # Start the server mlx_lm.server --model "rbizzoni/Velvet2-3B-MLX-8bit" # Calling the OpenAI-compatible server with curl curl -X POST "http://localhost:8080/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "rbizzoni/Velvet2-3B-MLX-8bit", "messages": [ {"role": "user", "content": "Hello"} ] }' - Hermes Agent
How to use rbizzoni/Velvet2-3B-MLX-8bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "rbizzoni/Velvet2-3B-MLX-8bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default rbizzoni/Velvet2-3B-MLX-8bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use rbizzoni/Velvet2-3B-MLX-8bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "rbizzoni/Velvet2-3B-MLX-8bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "rbizzoni/Velvet2-3B-MLX-8bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 7,421 Bytes
e5d5367 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | import json
import re
import uuid
from collections.abc import Sequence
from typing import Any, Optional
# --- Imports compatibili con vLLM vecchi (<= ~0.11) e nuovi (0.12+ / 0.19.x) ---
try: # vLLM nuovi: protocol diviso per endpoint
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
except ImportError: # vLLM vecchi: protocol unico
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest,
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
try: # vLLM nuovi: tool parser spostati in vllm.tool_parsers
from vllm.tool_parsers.abstract_tool_parser import (
ToolParser,
ToolParserManager,
)
except ImportError: # vLLM vecchi
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
ToolParser,
ToolParserManager,
)
try:
from vllm.utils import random_uuid
except ImportError: # vllm.utils riorganizzato in alcune versioni
def random_uuid() -> str:
return uuid.uuid4().hex
START_TAG = "<tool_call>"
END_TAG = "</tool_call>"
def _parse_calls(payload: str, request: ChatCompletionRequest) -> Optional[list]:
"""Return the calls in a ``<tool_call>`` payload, or None if invalid.
Every call must name a function offered in ``request.tools`` and carry
its arguments as a JSON object.
"""
allowed_names = {tool.function.name for tool in request.tools or []}
try:
parsed = json.loads(payload)
except ValueError:
return None
calls = parsed if isinstance(parsed, list) else [parsed]
if not calls or not all(
isinstance(call, dict)
and isinstance(call.get("name"), str)
and call["name"] in allowed_names
and isinstance(call.get("arguments", {}), dict)
for call in calls
):
return None
return [
{"name": call["name"], "arguments": call.get("arguments", {})}
for call in calls
]
def _partial_tag_len(text: str) -> int:
"""Length of the longest suffix of ``text`` that starts ``START_TAG``."""
for size in range(min(len(text), len(START_TAG) - 1), 0, -1):
if START_TAG.startswith(text[-size:]):
return size
return 0
@ToolParserManager.register_module("velvet")
class VelvetToolParser(ToolParser):
"""Parse Velvet ``<tool_call>[...]</tool_call>`` output."""
_pattern = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
def __init__(self, tokenizer: Any, *args: Any, **kwargs: Any):
# *args/**kwargs: inoltra eventuali parametri aggiuntivi che le
# versioni più recenti di vLLM passano al costruttore.
super().__init__(tokenizer, *args, **kwargs)
# Characters of the generated text already streamed or consumed.
self._scan_pos = 0
def adjust_request(
self, request: ChatCompletionRequest
) -> ChatCompletionRequest:
# <tool_call> and </tool_call> are special tokens: keep them in the
# decoded text, otherwise the parser cannot find the call.
if request.tools and request.tool_choice != "none":
request.skip_special_tokens = False
return request
def extract_tool_calls(
self, model_output: str, request: ChatCompletionRequest
) -> ExtractedToolCallInformation:
match = self._pattern.search(model_output)
calls = _parse_calls(match.group(1), request) if match else None
if calls is None:
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
tool_calls = [
ToolCall(
type="function",
function=FunctionCall(
name=call["name"],
arguments=json.dumps(call["arguments"], ensure_ascii=False),
),
)
for call in calls
]
prefix = model_output[: match.start()].strip()
return ExtractedToolCallInformation(
tools_called=True, tool_calls=tool_calls, content=prefix or None
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage:
"""Stream plain text as it arrives; send each tool call whole.
Text inside ``<tool_call>...</tool_call>`` is held back until the
closing tag arrives. A valid block becomes one chunk of tool calls;
an invalid one is released as plain text, as in non-streaming mode.
"""
content = ""
new_calls = []
pos = self._scan_pos
while True:
start = current_text.find(START_TAG, pos)
if start == -1:
hold = _partial_tag_len(current_text[pos:])
content += current_text[pos : len(current_text) - hold]
pos = len(current_text) - hold
break
content += current_text[pos:start]
pos = start
end = current_text.find(END_TAG, start)
if end == -1:
break # wait for the closing tag
block_end = end + len(END_TAG)
calls = _parse_calls(
current_text[start + len(START_TAG) : end].strip(), request
)
if calls is None:
content += current_text[start:block_end]
else:
new_calls.extend(calls)
pos = block_end
# Release anything still held back when generation stops.
max_tokens = getattr(request, "max_completion_tokens", None) or getattr(
request, "max_tokens", None
)
eos_id = getattr(self.model_tokenizer, "eos_token_id", None)
finished = (eos_id is not None and eos_id in delta_token_ids) or (
max_tokens is not None and len(current_token_ids) >= max_tokens
)
if finished:
content += current_text[pos:]
pos = len(current_text)
self._scan_pos = pos
delta_calls = []
for call in new_calls:
arguments = json.dumps(call["arguments"], ensure_ascii=False)
self.current_tool_id += 1
self.prev_tool_call_arr.append(call)
self.streamed_args_for_tool.append(arguments)
delta_calls.append(
DeltaToolCall(
index=self.current_tool_id,
id=f"chatcmpl-tool-{random_uuid()}",
type="function",
function=DeltaFunctionCall(
name=call["name"], arguments=arguments
).model_dump(exclude_none=True),
)
)
# An empty delta (rather than None) keeps vLLM sending the final
# chunk with finish_reason while a block is still buffered.
return DeltaMessage(content=content or None, tool_calls=delta_calls) |