Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-4bit") config = load_config("mlx-community/clairic-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download clairic_runtime/image_ragged_rows.py from mlx-community/clairic-4bit: direct link, hf CLI and curl.
- Browser
- Download file 13.7 kB
-
https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/image_ragged_rows.py
- Command line
-
hf download hf://mlx-community/clairic-4bit/clairic_runtime/image_ragged_rows.py
-
curl -L -o image_ragged_rows.py https://huggingface.co/mlx-community/clairic-4bit/resolve/main/clairic_runtime/image_ragged_rows.py
13.7 kB
| from __future__ import annotations | |
| from collections.abc import Callable | |
| from typing import TYPE_CHECKING, Any | |
| if TYPE_CHECKING: | |
| from .api import PredictRequest | |
| from .backend import EncodedRequest | |
| import hashlib | |
| import math | |
| import re | |
| from copy import deepcopy | |
| from dataclasses import dataclass | |
| from itertools import pairwise | |
| import numpy as np | |
| from PIL import Image | |
| from . import image_table_rows, table_rows | |
| from .schema import MAX_LIST_ITEMS | |
| VIEW_ID = "ragged-table-v1" | |
| TARGET_GATE = "target_is_billed_item" | |
| TARGET_ROW = "target_row" | |
| _BILLED_SCOPE = re.compile( | |
| "\\b(?:line[ _-]?items?|billed\\s+(?:products?|services?|items?|goods))\\b", | |
| re.IGNORECASE, | |
| ) | |
| _FINANCIAL_SCOPE = re.compile( | |
| "\\b(?:tax(?:es)?|fees?|surcharges?|payments?)\\b", re.IGNORECASE | |
| ) | |
| TARGET_INSTRUCTION = "Image 1 is the complete source document. Image 2 isolates one physical row from image 1. Classify and read only the target row in image 2. Use image 1 to identify the table column labels. A column header is not a billed item. Do not select a neighboring row or count row positions. " | |
| class RaggedRows: | |
| rows: tuple[tuple[int, int], ...] | |
| def views(self, image: Image.Image) -> tuple[Image.Image, ...]: | |
| return tuple( | |
| ( | |
| image.crop( | |
| (0, max(0, top - 3), image.width, min(image.height, bottom + 3)) | |
| ) | |
| for (top, bottom) in self.rows | |
| ) | |
| ) | |
| def detect(image: Image.Image) -> RaggedRows | None: | |
| if image_table_rows.detect(image) is not None: | |
| return None | |
| if image.mode != "RGB" or image.width < 100 or image.height < 60: | |
| return None | |
| grey = np.asarray(image.convert("L")) | |
| if np.mean(grey < 245) > 0.22: | |
| return None | |
| ink = grey < 200 | |
| candidates = [] | |
| for index, (top, bottom) in enumerate(image_table_rows._runs(ink.any(axis=1))): | |
| height = bottom - top | |
| if height < 10: | |
| continue | |
| band = ink[top:bottom] | |
| columns = band.any(axis=0) | |
| separators = [] | |
| for left, right in image_table_rows._runs(band.sum(axis=0) >= 0.9 * height): | |
| if ( | |
| right - left > max(3, height * 0.18) | |
| or left < 3 | |
| or right > image.width - 3 | |
| or columns[left - 3 : left].any() | |
| or columns[right : right + 3].any() | |
| ): | |
| continue | |
| separators.append((left, right)) | |
| if len(separators) >= 2: | |
| candidates.append((index, top, bottom, separators, columns)) | |
| if len(candidates) < 2: | |
| return None | |
| if any((right[0] != left[0] + 1 for (left, right) in pairwise(candidates))): | |
| return None | |
| if len({len(row[3]) for row in candidates}) != 1: | |
| return None | |
| if any((row[4][row[3][-1][1] + 3 :].any() for row in candidates)): | |
| return None | |
| heights = np.array([bottom - top for (_, top, bottom, _, _) in candidates]) | |
| spacing = np.diff([row[1] for row in candidates]) | |
| if np.max(heights) > 1.2 * np.min(heights) or np.max(spacing) > 1.3 * np.min( | |
| spacing | |
| ): | |
| return None | |
| return RaggedRows(tuple(((row[1], row[2]) for row in candidates))) | |
| def _eligible(spec: dict[str, Any]) -> bool: | |
| item = spec.get("items") | |
| properties = item.get("properties", {}) if isinstance(item, dict) else {} | |
| (scope, *exclusions) = re.split( | |
| "\\bExclude\\b", spec.get("instructions", ""), maxsplit=1, flags=re.IGNORECASE | |
| ) | |
| if exclusions: | |
| if re.search( | |
| "\\b(?:not|never|\\w+n['’]t)(?:\\s+\\w+){0,2}\\s*$", scope, re.IGNORECASE | |
| ): | |
| return False | |
| (excluded, *later) = re.split("[.;\\n]", exclusions[0], maxsplit=1) | |
| if ( | |
| later | |
| and _FINANCIAL_SCOPE.search(later[0]) | |
| or ( | |
| _FINANCIAL_SCOPE.search(excluded) | |
| and re.search( | |
| "\\b(?:include|including|keep|retain|except|unless|not|no|none)\\b", | |
| excluded, | |
| re.IGNORECASE, | |
| ) | |
| ) | |
| ): | |
| return False | |
| return ( | |
| spec.get("type") == "list" | |
| and isinstance(item, dict) | |
| and (item.get("type") == "object") | |
| and (not table_rows.join_wrapped_descriptions(spec.get("instructions", ""))) | |
| and (not spec.get("knowledge")) | |
| and (len(properties) >= 3) | |
| and bool(_BILLED_SCOPE.search(scope)) | |
| and (not _FINANCIAL_SCOPE.search(scope)) | |
| and all( | |
| ( | |
| child.get("type") in ("text", "number") and (not child.get("knowledge")) | |
| for child in properties.values() | |
| ) | |
| ) | |
| ) | |
| def row_fields(spec: dict[str, Any]) -> dict[str, dict[str, Any]]: | |
| wrapped = table_rows.join_wrapped_descriptions(spec.get("instructions", "")) | |
| membership = ( | |
| "A logical item can span several printed bands. Include it exactly once, at its first description band. Return false for a later continuation band belonging to that preceding item, even when its numeric cells are printed on the continuation band. Distinct separate source rows remain separate even if they repeat. " | |
| if wrapped | |
| else "" | |
| ) | |
| readout = ( | |
| "For the target item's first band, use the full document to read its complete wrapped description and cells from its continuation bands. Never borrow from a neighboring logical item. " | |
| if wrapped | |
| else "" | |
| ) | |
| return { | |
| TARGET_GATE: { | |
| "type": "boolean", | |
| "nullable": False, | |
| "instructions": TARGET_INSTRUCTION | |
| + membership | |
| + "Is the target row a billed product or service row? Return false for the column header, totals, tax, fees, or payment rows. " | |
| + spec.get("instructions", ""), | |
| }, | |
| TARGET_ROW: { | |
| "type": "object", | |
| "nullable": False, | |
| "properties": deepcopy(spec["items"]["properties"]), | |
| "instructions": TARGET_INSTRUCTION | |
| + readout | |
| + "Read the complete printed target row as an object. Whether it belongs in the requested list is decided separately. Always return an object with every requested cell. Missing cells and absent columns are null. Do not relabel an Amount column as Unit price. Never compute missing values.", | |
| }, | |
| } | |
| def prepare( | |
| request: PredictRequest, | |
| encoded: EncodedRequest, | |
| encode: Callable[..., EncodedRequest], | |
| *, | |
| scale: float = 2, | |
| ) -> EncodedRequest: | |
| (record, forecasts, tasks, typed, eos) = encoded | |
| annotations = record["_meta"]["extraction_contract"]["fields"] | |
| parents = [ | |
| name | |
| for (name, spec) in request.fields.items() | |
| if _eligible(spec) and annotations.get(name, {}).get("image_row_fields") | |
| ] | |
| if not parents or len(request.images) != 1 or request.documents: | |
| return encoded | |
| originals = record.get("_original_images") | |
| image = originals[0] if originals else request.images[0].to_pil() | |
| plan = detect(image) | |
| if plan is None: | |
| return encoded | |
| views = [] | |
| for view in plan.views(image): | |
| factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5) | |
| if factor > 1: | |
| view = view.resize( | |
| (int(view.width * factor), int(view.height * factor)), | |
| Image.Resampling.BICUBIC, | |
| ) | |
| views.append(view) | |
| salts = dict(record.get("_ar_task_salts", {})) | |
| gates = list(record.get("_source_row_gates", [])) | |
| changed = False | |
| for name in parents: | |
| (spec, annotation) = (request.fields[name], annotations[name]) | |
| previous = annotation["image_row_fields"] | |
| maximum = spec.get("max_items", MAX_LIST_ITEMS) | |
| if len(views) > maximum + 1 or len(views) > len(previous): | |
| continue | |
| (replacements, pairs) = ([], []) | |
| for index, (view, band) in enumerate(zip(views, plan.rows, strict=True)): | |
| child = previous[index] | |
| digest = hashlib.sha256(child.encode()).hexdigest()[:16] | |
| gate = "ragged_gate_" + digest | |
| while gate in record["fields"]: | |
| gate += "_" | |
| row_request = request.model_copy(update={"fields": row_fields(spec)}) | |
| (row_record, _, row_tasks, _, _) = encode( | |
| row_request, | |
| physical_row_view=True, | |
| image_views=lambda view=view: (image, view), | |
| image_view_key=f"{VIEW_ID}:{band}:scale{scale}", | |
| ) | |
| assert len(row_tasks) == 2 | |
| names = {TARGET_GATE: gate, TARGET_ROW: child} | |
| for source, ids, payload in row_tasks: | |
| target = names[source] | |
| record["fields"][target] = row_record["fields"][source] | |
| record["_ar_constraints"][target] = row_record["_ar_constraints"][ | |
| source | |
| ] | |
| salts[target] = row_record["_cache_salt"] | |
| annotations[target] = {"parent": name} | |
| replacements.append((target, ids, payload)) | |
| gates.append(gate) | |
| pairs.append((gate, child)) | |
| tasks = [ | |
| task for task in tasks if task[0] not in {name, *previous} | |
| ] + replacements | |
| retained = {copy for (_, copy) in pairs} | |
| for child in set(previous) - retained: | |
| record["fields"].pop(child, None) | |
| record["_ar_constraints"].pop(child, None) | |
| annotations.pop(child, None) | |
| salts.pop(child, None) | |
| annotation.pop("image_row_fields") | |
| annotation["ragged_row_fields"] = pairs | |
| annotation["ragged_row_max_items"] = maximum | |
| record["fields"][name].pop("_image_row_inventory", None) | |
| changed = True | |
| if not changed: | |
| return encoded | |
| record.update( | |
| _ar_tasks=tasks, | |
| _ar_task_salts=salts, | |
| _source_row_gates=gates, | |
| _ar_prefix=[], | |
| _ar_group_prefixes=[], | |
| ) | |
| record.pop("_ar_media_groups", None) | |
| return (record, forecasts, tasks, typed, eos) | |
| def _score(value: object, *, probability: bool = False) -> bool: | |
| return ( | |
| isinstance(value, (int, float)) | |
| and (not isinstance(value, bool)) | |
| and math.isfinite(value) | |
| and (0 <= value <= 1 if probability else value <= 0) | |
| ) | |
| def assemble( | |
| record: dict[str, Any], predictions: dict[str, dict[str, Any]] | |
| ) -> dict[str, dict[str, Any]]: | |
| result = {} | |
| for name, annotation in record["_meta"]["extraction_contract"]["fields"].items(): | |
| pairs = annotation.get("ragged_row_fields") | |
| if not pairs: | |
| continue | |
| rows = [(predictions[gate], predictions[copy]) for (gate, copy) in pairs] | |
| selected = [(gate, copy) for (gate, copy) in rows if gate.get("value") is True] | |
| complete = all((gate.get("complete", False) for (gate, _) in rows)) and all( | |
| (copy.get("complete", False) for (_, copy) in selected) | |
| ) | |
| exclusion_scored = all( | |
| (_score(gate.get("p_excluded"), probability=True) for (gate, _) in rows) | |
| ) | |
| relevant = [gate for (gate, _) in rows] + [copy for (_, copy) in selected] | |
| scored = exclusion_scored and all( | |
| (_score(answer.get("log_probability")) for answer in relevant) | |
| ) | |
| probability = ( | |
| sum((answer["log_probability"] for answer in relevant)) if scored else None | |
| ) | |
| if probability is not None and (not math.isfinite(probability)): | |
| (scored, probability) = (False, None) | |
| valid = ( | |
| complete | |
| and scored | |
| and all( | |
| ( | |
| gate.get("valid", False) and type(gate.get("value")) is bool | |
| for (gate, _) in rows | |
| ) | |
| ) | |
| and all( | |
| ( | |
| copy.get("valid", False) and isinstance(copy.get("value"), dict) | |
| for (_, copy) in selected | |
| ) | |
| ) | |
| ) | |
| within_limit = len(selected) <= annotation["ragged_row_max_items"] | |
| valid &= within_limit | |
| answer = { | |
| "value": [copy["value"] for (_, copy) in selected], | |
| "valid": valid, | |
| "complete": complete, | |
| "confidence": math.exp(probability) | |
| if valid and probability is not None | |
| else 0.0, | |
| "confidence_method": "ar_physical_image_row_eligibility_and_copy_probability", | |
| "confidences": [ | |
| math.exp(gate["log_probability"] + copy["log_probability"]) | |
| if scored | |
| else 0.0 | |
| for (gate, copy) in selected | |
| ], | |
| "p_null": math.prod((gate["p_excluded"] for (gate, _) in rows)) | |
| if exclusion_scored | |
| else 0.0, | |
| "p_null_method": "product_physical_image_row_exclusion_probabilities" | |
| if exclusion_scored | |
| else "unavailable_physical_image_row_exclusion_probability", | |
| "p_nulls": [ | |
| gate["p_excluded"] if exclusion_scored else 0.0 | |
| for (gate, _) in selected | |
| ], | |
| "spans": [None] * len(selected), | |
| } | |
| if probability is not None: | |
| answer["log_probability"] = probability | |
| if not valid: | |
| answer["failure_reason"] = ( | |
| "ragged_row_output_exceeds_max_items" | |
| if not within_limit | |
| else "ragged_row_unscored" | |
| if not scored | |
| else "ragged_row_read_incomplete" | |
| ) | |
| result[name] = answer | |
| return result | |