Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-4bit") config = load_config("mlx-community/clairic-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 6,856 Bytes
1a48d6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | """Clairic request and response schemas."""
from __future__ import annotations
import base64
import io
from dataclasses import dataclass
from typing import Any, LiteralString, cast
from .embedded import (
MAX_IMAGE_BYTES,
MAX_IMAGES,
MAX_IMAGES_BYTES,
EmbeddedImageError,
decoded_base64_size,
normalize_embedded_image,
validate_image_count,
validate_total_image_bytes,
)
from PIL import Image, ImageOps
from pydantic import (
BaseModel,
ConfigDict,
Field,
StrictStr,
field_validator,
model_validator,
)
from pydantic_core import PydanticCustomError
from .documents import ClairDocument
MODEL_ID = "clairic"
def _image_error(error: EmbeddedImageError) -> PydanticCustomError:
# Every code and message originates from fixed literals in the shared helper.
return PydanticCustomError(
cast(LiteralString, error.code), cast(LiteralString, str(error))
)
class ClairImage(BaseModel):
"""An embedded PNG, JPEG, or WebP image validated before it reaches the processor."""
model_config = ConfigDict(extra="forbid", frozen=True)
content_type: StrictStr
base64: StrictStr
rotation: int = 0
@model_validator(mode="before")
@classmethod
def parse_image(cls, value: object) -> object:
try:
rotation = 0
explicit_rotation = isinstance(value, dict) and "rotation" in value
if explicit_rotation:
value = dict(value) # type: ignore[call-overload] # explicit_rotation guards the dict input.
rotation = value.pop("rotation") # type: ignore[attr-defined]
if type(rotation) is not int or rotation not in (0, 90, 180, 270):
raise EmbeddedImageError(
"invalid_image",
"rotation must be 0, 90, 180 or 270 clockwise degrees",
)
normalized = normalize_embedded_image(
value, strict=True, max_image_bytes=MAX_IMAGE_BYTES
)
return (
{**normalized, "rotation": rotation}
if explicit_rotation
else normalized
)
except EmbeddedImageError as error:
raise _image_error(error) from error
def to_pil(self) -> Image.Image:
from .orientation import orient_document
with Image.open(io.BytesIO(base64.b64decode(self.base64))) as image:
result = ImageOps.exif_transpose(image)
if "A" in result.getbands() or "transparency" in result.info:
# Render visible document pixels before discarding alpha. Hidden
# RGB values can otherwise erase opaque black text completely.
rgba = result if result.mode == "RGBA" else result.convert("RGBA")
result = Image.new("RGB", rgba.size, "white")
result.paste(rgba, mask=rgba.getchannel("A"))
else:
result = result.convert("RGB")
if "rotation" in self.model_fields_set:
return (
result.rotate(-self.rotation, expand=True)
if self.rotation
else result
)
return orient_document(result)
@property
def decoded_size(self) -> int:
return decoded_base64_size(self.base64)
class PredictOptions(BaseModel):
model_config = ConfigDict(extra="forbid")
include_quantiles: bool = False
return_state_text: bool = False
class PredictRequest(BaseModel):
"""A state (any text), optional images, and any number of typed fields. Field specs are checked against
the field schema when the request is encoded."""
model_config = ConfigDict(extra="forbid")
model: StrictStr | None = None
state: StrictStr = ""
images: tuple[ClairImage, ...] = Field(
default=(), json_schema_extra={"maxItems": MAX_IMAGES}
)
documents: tuple[ClairDocument, ...] = Field(default=(), max_length=MAX_IMAGES)
fields: dict[StrictStr, dict[StrictStr, Any]]
reference_date: StrictStr | None = None
options: PredictOptions = Field(default_factory=PredictOptions)
@field_validator("images", mode="before")
@classmethod
def check_image_count(cls, value: object) -> object:
try:
return validate_image_count(value, max_images=MAX_IMAGES)
except EmbeddedImageError as error:
raise _image_error(error) from error
@model_validator(mode="after")
def check_content(self) -> PredictRequest:
if not self.fields:
raise PydanticCustomError(
"empty_fields", "fields must declare at least one field"
)
if not self.state.strip() and not self.images and not self.documents:
raise PydanticCustomError(
"empty_state", "state must be non-empty when no images are sent"
)
try:
validate_total_image_bytes(
(
decoded_base64_size(item.base64)
for item in (*self.images, *self.documents)
),
max_bytes=MAX_IMAGES_BYTES, # type: ignore[attr-defined] # Both validated media types have base64.
)
except EmbeddedImageError as error:
raise _image_error(error) from error
return self
@dataclass(frozen=True, slots=True)
class PredictOutput:
answers: dict[str, dict[str, Any]]
truncated: bool
state_tokens: int
image_tokens: int
schema_tokens: int
input_tokens: int
passes: int
state_text: str | None = None
output_tokens: int = 0
def _without_quantiles(answer: dict[str, Any]) -> dict[str, Any]:
answer = {key: value for key, value in answer.items() if key != "quantiles"}
if isinstance(answer.get("steps"), list):
answer["steps"] = [_without_quantiles(step) for step in answer["steps"]]
return answer
def serialize_response(
request: PredictRequest, output: PredictOutput, model_id: str
) -> dict[str, Any]:
answers = {
name: answer
if request.options.include_quantiles
else _without_quantiles(answer)
for name, answer in output.answers.items()
}
content: dict[str, Any] = {
"model": model_id,
"answers": answers,
"truncated": output.truncated,
"usage": {
"state_tokens": output.state_tokens,
"image_tokens": output.image_tokens,
"schema_tokens": output.schema_tokens,
"input_tokens": output.input_tokens,
"output_tokens": output.output_tokens,
"passes": output.passes,
},
}
if request.options.return_state_text and output.state_text is not None:
content["state_text"] = output.state_text
return content
|