Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-4bit") config = load_config("mlx-community/clairic-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download clairic_runtime/grounding.py from mlx-community/clairic-4bit: direct link, hf CLI and curl.
- Browser
- Download file 16 kB
-
https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/grounding.py
- Command line
-
hf download hf://mlx-community/clairic-4bit/clairic_runtime/grounding.py
-
curl -L -o grounding.py https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/grounding.py
16 kB
| from __future__ import annotations | |
| import json | |
| import re | |
| import unicodedata | |
| from decimal import Decimal, InvalidOperation, localcontext | |
| from typing import Any | |
| from . import ar | |
| MAX_GRAMMAR_CHARACTERS = 4096 | |
| _STATE_ONLY = re.compile( | |
| "\\b(?:from (?:the )?(?:accompanying (?:state )?|state )text only|supplied in the accompanying state text)\\b", | |
| re.IGNORECASE, | |
| ) | |
| _NULL_ABSENCE_GUARD = re.compile( | |
| "(?:^|(?<=[.;\\n]))\\s*return\\s+null\\s+(?:only\\s+)?if\\s+(?:not\\s+(?:explicitly\\s+)?printed|absent|missing)\\s*(?:[.;]|$)", | |
| re.IGNORECASE, | |
| ) | |
| _EXCLUDED_SOURCE = re.compile( | |
| "\\b(?:not|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE | |
| ) | |
| _DOCUMENT = "(?:(?:this|the)(?:\\s+(?:supplied|provided))?|supplied|provided)\\s+(?:note|document|text)" | |
| _DOCUMENT_ONLY = re.compile( | |
| f"\\b(?:use\\s+(?:only\\s+{_DOCUMENT}|{_DOCUMENT}\\s+only)|from\\s+{_DOCUMENT}\\s+only)\\b", | |
| re.IGNORECASE, | |
| ) | |
| _DOCUMENT_EXCLUDED = re.compile( | |
| "\\b(?:not|never|don't|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE | |
| ) | |
| _DERIVED_NUMBER = re.compile( | |
| "\\b(?:calculat\\w*|comput\\w*|deriv\\w*|convert\\w*|conversion\\w*|count\\w*|sum\\w*|averag\\w*|mean|median|difference|ratio|growth|product|multipli\\w*|multiply|subtract\\w*|minus|divid\\w*|division|quotient|remainder|add|adding|addition|plus|times|percent\\w*|proportion|fraction)\\b|\\b(?:how many|how much|total number of|number of (?:line )?(?:items|rows|entries|records))\\b|[+*/\\u00d7\\u00f7=]|\\s[-\\u2212]\\s", | |
| re.IGNORECASE, | |
| ) | |
| _CURRENCY_CODES = frozenset( | |
| { | |
| "USD", | |
| "EUR", | |
| "GBP", | |
| "JPY", | |
| "CNY", | |
| "INR", | |
| "AUD", | |
| "CAD", | |
| "CHF", | |
| "HKD", | |
| "NZD", | |
| "SGD", | |
| "SEK", | |
| "NOK", | |
| "DKK", | |
| "KRW", | |
| "BRL", | |
| "MXN", | |
| "ZAR", | |
| } | |
| ) | |
| _EXACT_TEXT = re.compile( | |
| "\\b(?:verbatim|raw|literal\\w*|print\\w*|exact\\w*|characters?|digits?|whitespace|spacing|newlines?)\\b|\\b(?:line\\s+breaks?|white\\s+space|do\\s+not\\s+normaliz\\w*)\\b", | |
| re.IGNORECASE, | |
| ) | |
| def allows_whitespace_normalization(spec: dict[str, Any]) -> bool: | |
| return ( | |
| spec.get("type") == "text" | |
| and (not spec.get("knowledge")) | |
| and ("source_only" not in spec) | |
| and (not spec.get("_physical_row_scope")) | |
| and ( | |
| not any( | |
| (key in spec for key in ("enum", "pattern", "minLength", "maxLength")) | |
| ) | |
| ) | |
| and (not _EXACT_TEXT.search(spec.get("instructions", ""))) | |
| ) | |
| def collapse_whitespace(source: str) -> tuple[str, list[tuple[int, int]]]: | |
| (characters, offsets) = ([], []) | |
| for match in re.finditer("\\s+|\\S", source): | |
| characters.append(" " if match[0].isspace() else match[0]) | |
| offsets.append(match.span()) | |
| return ("".join(characters), offsets) | |
| def state_only(spec: dict[str, Any]) -> bool: | |
| instructions = spec.get("instructions", "") | |
| scope = _NULL_ABSENCE_GUARD.sub("", instructions) | |
| return ( | |
| spec["type"] in ("text", "number") | |
| and (not spec.get("knowledge")) | |
| and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE)) | |
| and any( | |
| ( | |
| not _EXCLUDED_SOURCE.search(instructions[: match.start()]) | |
| for match in _STATE_ONLY.finditer(instructions) | |
| ) | |
| ) | |
| ) | |
| def explicit_media_scope(spec: dict[str, Any]) -> bool: | |
| return ( | |
| spec.get("type") in ("text", "number") | |
| and (not spec.get("knowledge")) | |
| and bool( | |
| re.search( | |
| "\\b(?:page|image)\\s+\\d+\\b", | |
| spec.get("instructions", ""), | |
| re.IGNORECASE, | |
| ) | |
| ) | |
| ) | |
| def document_only(spec: dict[str, Any]) -> bool: | |
| instructions = spec.get("instructions", "") | |
| scope = _NULL_ABSENCE_GUARD.sub("", instructions) | |
| return ( | |
| spec["type"] in ("text", "number") | |
| and (not spec.get("knowledge")) | |
| and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE)) | |
| and (not (spec["type"] == "number" and _DERIVED_NUMBER.search(scope))) | |
| and any( | |
| ( | |
| not _DOCUMENT_EXCLUDED.search(instructions[: match.start()]) | |
| for match in _DOCUMENT_ONLY.finditer(instructions) | |
| ) | |
| ) | |
| ) | |
| def applies(spec: dict[str, Any], media: object | None) -> bool: | |
| return ( | |
| spec["type"] in ("text", "number") | |
| and media is None | |
| and (not spec.get("knowledge", False)) | |
| and ( | |
| spec["type"] == "text" | |
| or spec.get("source_only", False) | |
| or state_only(spec) | |
| or document_only(spec) | |
| ) | |
| ) | |
| def _document_number_span(text: str, span: tuple[int, int]) -> bool: | |
| (start, _) = span | |
| left = start | |
| while left and (text[left - 1].isalnum() or text[left - 1] in "_./-"): | |
| left -= 1 | |
| prefix = text[left:start].lstrip("_./-") | |
| return not prefix or not prefix[0].isalpha() or prefix.upper() in _CURRENCY_CODES | |
| def _text_states(source: str, prefix: str) -> list[str]: | |
| transitions: list[dict[str, int]] | |
| (transitions, links, lengths) = ([{}], [-1], [0]) | |
| last = 0 | |
| for char in source: | |
| current = len(transitions) | |
| transitions.append({}) | |
| lengths.append(lengths[last] + 1) | |
| links.append(0) | |
| parent = last | |
| while parent >= 0 and char not in transitions[parent]: | |
| transitions[parent][char] = current | |
| parent = links[parent] | |
| if parent >= 0: | |
| target = transitions[parent][char] | |
| if lengths[parent] + 1 == lengths[target]: | |
| links[current] = target | |
| else: | |
| clone = len(transitions) | |
| transitions.append(dict(transitions[target])) | |
| lengths.append(lengths[parent] + 1) | |
| links.append(links[target]) | |
| while parent >= 0 and transitions[parent].get(char) == target: | |
| transitions[parent][char] = clone | |
| parent = links[parent] | |
| links[target] = links[current] = clone | |
| last = current | |
| quote = json.dumps('"') | |
| rules = [] | |
| for index, edges in enumerate(transitions): | |
| choices = [quote] if index else [] | |
| for char, target in edges.items(): | |
| escaped = json.dumps(char, ensure_ascii=False)[1:-1] | |
| choices.append( | |
| json.dumps(escaped, ensure_ascii=False) + f" {prefix}{target}" | |
| ) | |
| rules.append(f"{prefix}{index} ::= " + " | ".join(choices)) | |
| return rules | |
| def text_grammar( | |
| source: str, *, nullable: bool = True, normalized_source: str | None = None | |
| ) -> str: | |
| if not source: | |
| if not nullable: | |
| raise ValueError("Empty source permits no non-null text value") | |
| return 'root ::= "null"' | |
| quote = json.dumps('"') | |
| starts = ['"null"'] if nullable else [] | |
| starts.append(quote + " s0") | |
| states = _text_states(source, "s") | |
| if normalized_source and normalized_source != source: | |
| starts.append(quote + " n0") | |
| states.extend(_text_states(normalized_source, "n")) | |
| return "\n".join(["root ::= " + " | ".join(starts), *states]) | |
| def numbers(source: str) -> dict[Decimal, list[tuple[int, int]]]: | |
| from .model import NUMERAL_RUN, numeral_convention, numeral_values | |
| convention = numeral_convention(source) | |
| result: dict[Decimal, list[tuple[int, int]]] = {} | |
| consumed_end = 0 | |
| for run in NUMERAL_RUN.finditer(source): | |
| if run.start() < consumed_end: | |
| continue | |
| readings = [(run.group(), run.start(), run.end())] | |
| if not numeral_values(run.group(), convention): | |
| readings = [ | |
| (m.group(), run.start() + m.start(), run.start() + m.end()) | |
| for m in re.finditer("[^, \\u00a0\\u202f\\u2009\\u2007]+", run.group()) | |
| ] | |
| for printed, left, end in readings: | |
| values = numeral_values(printed, convention) | |
| if not values: | |
| continue | |
| exponent = re.match("[eE]([-+]?\\d+)", source[end:]) | |
| if exponent: | |
| consumed_end = end + len(exponent[0]) | |
| if len(exponent[1]) > 4 or abs(int(exponent[1])) > 308: | |
| continue | |
| with localcontext() as context: | |
| context.prec = max(context.prec, len(printed) + 310) | |
| values = [ | |
| value * Decimal(10) ** int(exponent[1]) for value in values | |
| ] | |
| end += len(exponent[0]) | |
| start = left | |
| while start > max(0, left - 32) and ( | |
| source[start - 1] in " \t-+−" | |
| or unicodedata.category(source[start - 1]) == "Sc" | |
| ): | |
| start -= 1 | |
| signed = source[start:left].strip() | |
| negative = any((c in signed for c in "-−")) and ( | |
| start == 0 or not source[start - 1].isalnum() | |
| ) | |
| left = start + len(source[start:left]) - len(source[start:left].lstrip()) | |
| before = source[max(0, left - 32) : left] | |
| after = source[end : end + 64] | |
| (prefix, suffix) = (before.rstrip(), after.lstrip()) | |
| accounting = ( | |
| any( | |
| ( | |
| c in ".," or unicodedata.category(c) == "Sc" | |
| for c in printed + signed | |
| ) | |
| ) | |
| or not suffix[1:].split("\n")[0].strip() | |
| ) | |
| if prefix.endswith("(") and suffix.startswith(")") and accounting: | |
| negative = True | |
| left -= len(before) - len(prefix) + 1 | |
| end += len(after) - len(suffix) + 1 | |
| for value in values: | |
| if negative: | |
| value = value.copy_negate() | |
| numeric = ( | |
| int(value) if value == value.to_integral_value() else float(value) | |
| ) | |
| if Decimal(str(numeric)) != value: | |
| continue | |
| result.setdefault(value, []).append((left, end)) | |
| return result | |
| class Source: | |
| def __init__(self, text: str, *, normalize_whitespace: bool = False) -> None: | |
| self.text = text | |
| self.numerals = numbers(text) | |
| self.normalize_whitespace = normalize_whitespace | |
| self._normalized_source: tuple[str, list[tuple[int, int]]] | None = None | |
| self._grammars: dict[tuple[bool, bool], str] = {} | |
| self._constraints: dict[tuple[str, bool, bool], dict[str, str]] = {} | |
| def whitespace_normalization(self, spec: dict[str, Any]) -> bool: | |
| return self.normalize_whitespace and allows_whitespace_normalization(spec) | |
| def _whitespace_source(self) -> tuple[str, list[tuple[int, int]]]: | |
| if self._normalized_source is None: | |
| self._normalized_source = collapse_whitespace(self.text) | |
| return self._normalized_source | |
| def normalized_span(self, value: object) -> tuple[bool, list[int] | None]: | |
| if not isinstance(value, str) or not value: | |
| return (False, None) | |
| matches = set() | |
| start = self.text.find(value) | |
| while start >= 0: | |
| matches.add((start, start + len(value))) | |
| start = self.text.find(value, start + 1) | |
| (normalized, offsets) = self._whitespace_source() | |
| start = normalized.find(value) | |
| while start >= 0: | |
| matches.add((offsets[start][0], offsets[start + len(value) - 1][1])) | |
| start = normalized.find(value, start + 1) | |
| return (bool(matches), list(next(iter(matches))) if len(matches) == 1 else None) | |
| def span( | |
| self, value: object, *, spec: dict[str, Any] | None = None | |
| ) -> tuple[bool, list[int] | None]: | |
| if isinstance(value, str): | |
| if spec is not None and self.whitespace_normalization(spec): | |
| return self.normalized_span(value) | |
| if not value: | |
| return (False, None) | |
| start = self.text.find(value) | |
| if start < 0: | |
| return (False, None) | |
| unique = bool(value) and self.text.find(value, start + 1) < 0 | |
| return (True, [start, start + len(value)] if unique else None) | |
| try: | |
| number = Decimal(str(value)) | |
| except InvalidOperation: | |
| return (False, None) | |
| if not number.is_finite(): | |
| return (False, None) | |
| matches = self.numerals.get(number, []) | |
| return (bool(matches), list(matches[0]) if len(matches) == 1 else None) | |
| def constraint(self, spec: dict[str, Any]) -> dict[str, str]: | |
| document_numbers = ( | |
| spec["type"] == "number" | |
| and document_only(spec) | |
| and (not spec.get("source_only")) | |
| and (not state_only(spec)) | |
| ) | |
| key = ( | |
| json.dumps(ar.schema(spec), ensure_ascii=False), | |
| bool(document_numbers), | |
| self.whitespace_normalization(spec), | |
| ) | |
| if key not in self._constraints: | |
| self._constraints[key] = self._constraint( | |
| spec, document_numbers=document_numbers | |
| ) | |
| return self._constraints[key] | |
| def _constraint( | |
| self, spec: dict[str, Any], *, document_numbers: bool = False | |
| ) -> dict[str, str]: | |
| from jsonschema import Draft202012Validator | |
| schema = ar.schema(spec) | |
| if spec["type"] == "number" or "enum" in schema: | |
| numerals = self.numerals | |
| if document_numbers: | |
| numerals = { | |
| value: spans | |
| for (value, spans) in numerals.items() | |
| if any((_document_number_span(self.text, span) for span in spans)) | |
| } | |
| candidates = schema.get("enum", list(numerals)) | |
| validator = Draft202012Validator(schema) | |
| values = [] | |
| for candidate in candidates: | |
| value = candidate | |
| if isinstance(value, Decimal): | |
| value = ( | |
| int(value) | |
| if value == value.to_integral_value() | |
| else float(value) | |
| ) | |
| if ( | |
| value is not None | |
| and validator.is_valid(value) | |
| and (not document_numbers or Decimal(str(value)) in numerals) | |
| and self.span(value)[0] | |
| ): | |
| values.append(value) | |
| if validator.is_valid(None): | |
| values.append(None) | |
| if not values: | |
| raise ValueError( | |
| "Field constraints permit no literal source value or null" | |
| ) | |
| if spec["type"] == "number": | |
| literals = [ | |
| "null" if value is None else str(Decimal(str(value))) | |
| for value in values | |
| ] | |
| return { | |
| "ebnf": "root ::= " | |
| + " | ".join((json.dumps(value) for value in literals)) | |
| } | |
| return { | |
| "json_schema": json.dumps( | |
| {**schema, "enum": values}, ensure_ascii=False | |
| ) | |
| } | |
| if len(self.text) > MAX_GRAMMAR_CHARACTERS or any( | |
| (key in schema for key in ("minLength", "maxLength", "pattern")) | |
| ): | |
| return {"json_schema": json.dumps(schema, ensure_ascii=False)} | |
| nullable = spec.get("nullable", True) | |
| normalize = self.whitespace_normalization(spec) | |
| key = (nullable, normalize) | |
| if key not in self._grammars: | |
| normalized = self._whitespace_source()[0] if normalize else None | |
| self._grammars[key] = text_grammar( | |
| self.text, nullable=nullable, normalized_source=normalized | |
| ) | |
| return {"ebnf": self._grammars[key]} | |