Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-flash-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-flash-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-flash-4bit") config = load_config("mlx-community/clairic-flash-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-flash-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-flash-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-flash-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-flash-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-flash-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-flash-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download clairic_runtime/po_image_supplement.py from mlx-community/clairic-flash-4bit: direct link, hf CLI and curl.
- Browser
- Download file 8.2 kB
-
https://huggingface.co/mlx-community/clairic-flash-4bit/resolve/main/clairic_runtime/po_image_supplement.py
- Command line
-
hf download hf://mlx-community/clairic-flash-4bit/clairic_runtime/po_image_supplement.py
-
curl -L -o po_image_supplement.py https://huggingface.co/mlx-community/clairic-flash-4bit/resolve/main/clairic_runtime/po_image_supplement.py
8.2 kB
| from __future__ import annotations | |
| import base64 | |
| import io | |
| import json | |
| from typing import TYPE_CHECKING | |
| from .api import ClairImage | |
| from . import ar, grounding, identifier_spelling, image_focus | |
| from PIL import Image | |
| if TYPE_CHECKING: | |
| from collections.abc import Callable, Sequence | |
| from typing import Any | |
| from .api import PredictRequest | |
| from .backend import ARTask, EncodedRequest | |
| from transformers import PreTrainedTokenizerBase | |
| VERSION = "27b-po-all-content-horizontal2-supplement-v2" | |
| MAX_PIXELS = 1600000 | |
| def supplement(images: Sequence[Image.Image]) -> ClairImage: | |
| views: list[Image.Image] = [] | |
| for image in images: | |
| focused = image_focus.focus_document(image) | |
| views.append( | |
| focused.resize( | |
| (round(focused.width * 2.0), focused.height), Image.Resampling.BICUBIC | |
| ) | |
| ) | |
| board = Image.new( | |
| "RGB", | |
| ( | |
| max((image.width for image in views)), | |
| sum((image.height for image in views)) + 32 * (len(views) - 1), | |
| ), | |
| "white", | |
| ) | |
| y = 0 | |
| for image in views: | |
| board.paste(image, (0, y)) | |
| y += image.height + 32 | |
| scale = min(1.0, (MAX_PIXELS / (board.width * board.height)) ** 0.5) | |
| if scale < 1: | |
| board = board.resize( | |
| (max(1, int(board.width * scale)), max(1, int(board.height * scale))), | |
| Image.Resampling.BICUBIC, | |
| ) | |
| buffer = io.BytesIO() | |
| board.save(buffer, format="PNG") | |
| return ClairImage.model_validate( | |
| { | |
| "content_type": "image/png", | |
| "base64": base64.b64encode(buffer.getvalue()).decode(), | |
| } | |
| ) | |
| def plain_task( | |
| task: ARTask, | |
| wire_spec: dict[str, Any], | |
| public_spec: dict[str, Any], | |
| tokenizer: PreTrainedTokenizerBase, | |
| max_length: int, | |
| ) -> tuple[ARTask, dict[str, Any], dict[str, str]]: | |
| (name, ids, media) = task | |
| source_ids = list(ids) | |
| text = tokenizer.decode( | |
| ids, skip_special_tokens=False, clean_up_tokenization_spaces=False | |
| ) | |
| if tokenizer.encode(text, add_special_tokens=False) != source_ids: | |
| raise ValueError( | |
| "PO scalar source prompt cannot make a lossless tokenizer roundtrip" | |
| ) | |
| before = identifier_spelling.INSTRUCTION + "\n" | |
| if text.count(before) != 1 or wire_spec.get("instructions", "").count(before) != 1: | |
| raise ValueError( | |
| "PO scalar source must contain exactly one spelling instruction" | |
| ) | |
| scalar = { | |
| **wire_spec, | |
| "instructions": wire_spec["instructions"].replace(before, ""), | |
| } | |
| for key in ("minLength", "maxLength", "pattern"): | |
| if key in public_spec: | |
| scalar[key] = public_spec[key] | |
| (old_schema, schema) = (ar.schema(wire_spec), ar.schema(scalar)) | |
| text = text.replace(before, "") | |
| if schema != old_schema: | |
| old = "\nSchema: " + json.dumps( | |
| old_schema, ensure_ascii=False, separators=(",", ":") | |
| ) | |
| new = "\nSchema: " + json.dumps( | |
| schema, ensure_ascii=False, separators=(",", ":") | |
| ) | |
| if text.count(old) != 1: | |
| raise ValueError("PO scalar source must contain exactly one native schema") | |
| text = text.replace(old, new) | |
| plain_ids = tokenizer.encode(text, add_special_tokens=False) | |
| if len(plain_ids) > max_length: | |
| raise ValueError("PO scalar detail exceeds the complete model context") | |
| return ((name, plain_ids, media), scalar, {"json_schema": json.dumps(schema)}) | |
| def selected_fields( | |
| request: PredictRequest, encoded: EncodedRequest | |
| ) -> dict[str, dict[str, Any]]: | |
| image_count = len(request.images) | |
| if image_count not in (1, 2, 3) or request.documents: | |
| return {} | |
| (record, _, tasks, _, _) = encoded | |
| contracts = record.get("_meta", {}).get("identifier_spelling", {}) | |
| active = {name for (name, _, media) in tasks if media is not None} | |
| return { | |
| name: request.fields[name] | |
| for (name, contract) in contracts.items() | |
| if name in active | |
| and contract.get("wire") == identifier_spelling.VERSION | |
| and (not request.fields[name].get("knowledge")) | |
| and (not grounding.state_only(request.fields[name])) | |
| } | |
| def prepare( | |
| request: PredictRequest, | |
| encoded: EncodedRequest, | |
| *, | |
| encode_child: Callable[[PredictRequest], EncodedRequest], | |
| tokenizer: PreTrainedTokenizerBase, | |
| max_length: int, | |
| media_groups: Callable[..., list[dict[str, Any]]], | |
| ) -> EncodedRequest: | |
| selected = selected_fields(request, encoded) | |
| if not selected: | |
| return encoded | |
| image_count = len(request.images) | |
| (record, forecasts, tasks, typed, eos) = encoded | |
| contracts = record["_meta"]["identifier_spelling"] | |
| originals = record.get("_original_images", ()) | |
| if len(originals) != image_count: | |
| raise ValueError("PO supplement must retain every original image in order") | |
| fields = image_focus.preserve_scalar_fields(selected) | |
| supplements = ( | |
| tuple((supplement([image]) for image in originals)) | |
| if image_count == 2 | |
| else (supplement(originals),) | |
| ) | |
| child_request = request.model_copy( | |
| update={"fields": fields, "images": (*request.images, *supplements)} | |
| ) | |
| (child, _, replacements, child_typed, child_eos) = encode_child(child_request) | |
| by_name = {task[0]: task for task in replacements} | |
| if ( | |
| child_typed is not None | |
| or child_eos != eos | |
| or set(by_name) != set(selected) | |
| or (len(replacements) != len(selected)) | |
| ): | |
| raise ValueError( | |
| "PO supplement must retain the complete selected scalar task contract" | |
| ) | |
| plain = image_count == 1 | |
| constraints: dict[str, dict[str, Any]] | |
| remaining: dict[str, Any] | list[str] | |
| (fields, constraints) = ({}, {}) | |
| for name, public_spec in selected.items(): | |
| if plain: | |
| (by_name[name], fields[name], constraints[name]) = plain_task( | |
| by_name[name], child["fields"][name], public_spec, tokenizer, max_length | |
| ) | |
| else: | |
| fields[name] = child["fields"][name] | |
| constraints[name] = child["_ar_constraints"][name] | |
| updated = [by_name.get(task[0], task) for task in tasks] | |
| salts = dict(record.get("_ar_task_salts", {})) | |
| for name in selected: | |
| record["fields"][name] = fields[name] | |
| record["_ar_constraints"][name] = constraints[name] | |
| salts[name] = child.get("_ar_task_salts", {}).get(name, child["_cache_salt"]) | |
| if plain: | |
| remaining = { | |
| name: contract | |
| for (name, contract) in contracts.items() | |
| if name not in selected | |
| } | |
| if remaining: | |
| record["_meta"]["identifier_spelling"] = remaining | |
| else: | |
| record["_meta"].pop("identifier_spelling", None) | |
| for key in ("model_document_scalar_view", "identifier_spelling_image_view"): | |
| previous = record["_meta"].get(key) | |
| if previous is None: | |
| continue | |
| remaining = [name for name in previous["fields"] if name not in selected] | |
| if not remaining: | |
| record["_meta"].pop(key) | |
| else: | |
| retained = {**previous, "fields": remaining} | |
| if "po_fields" in retained: | |
| retained["po_fields"] = [ | |
| name for name in retained["po_fields"] if name not in selected | |
| ] | |
| record["_meta"][key] = retained | |
| record["_meta"]["model_po_supplement"] = { | |
| "version": VERSION, | |
| "fields": list(selected), | |
| "original_image_count": image_count, | |
| "native_image_count": len(child_request.images), | |
| "horizontal_scale": 2.0, | |
| "supplement_max_pixels": MAX_PIXELS, | |
| "wire": "native_scalar" if plain else identifier_spelling.VERSION, | |
| } | |
| record.update( | |
| _ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[] | |
| ) | |
| record["_ar_media_groups"] = media_groups( | |
| updated, | |
| default_salt=record["_cache_salt"], | |
| task_salts=salts, | |
| annotations=record["_meta"]["extraction_contract"]["fields"], | |
| ) | |
| return (record, forecasts, updated, typed, eos) | |