Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/image_financial_lines.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/image_financial_lines.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/image_financial_lines.py
-
curl -L -o image_financial_lines.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/image_financial_lines.py
13.2 kB
| from __future__ import annotations | |
| from collections.abc import Callable | |
| from typing import TYPE_CHECKING, Any | |
| if TYPE_CHECKING: | |
| from .api import PredictRequest | |
| from .backend import EncodedRequest | |
| import math | |
| import re | |
| from copy import deepcopy | |
| import numpy as np | |
| from jsonschema import Draft202012Validator, ValidationError | |
| from PIL import Image | |
| from . import ar, image_ragged_rows, image_table_rows, printed_numbers | |
| from .schema import MAX_LIST_ITEMS | |
| LINE = { | |
| "type": "text", | |
| "nullable": False, | |
| "instructions": "Copy the complete single printed line verbatim, including all words, prefixes, punctuation and numbers. Do not extract a name or normalize any part. Return only the complete line.", | |
| } | |
| _TAX = re.compile("\\btax(?:es)?(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE) | |
| _FEE = re.compile( | |
| "\\b(?:fees?|surcharges?)(?:\\s+(?:refund|credit|adjustment))?$", re.IGNORECASE | |
| ) | |
| _SUMMARY = re.compile( | |
| "(?:sub[ -]?total|grand total|total|amount due|balance(?: due| forward)?|tax total|fee total)", | |
| re.IGNORECASE, | |
| ) | |
| _PAYMENT = re.compile( | |
| "^(?:payments?\\s+(?:received|instructions?|due|method|status)|(?:received|posted|applied)\\s+payments?)\\b|\\bpayments?$", | |
| re.IGNORECASE, | |
| ) | |
| _TRANSFORM = re.compile( | |
| "\\b(?:deduplicat\\w*|unique|aggregat\\w*|summari[sz]\\w*|combin\\w*|group\\w*|averag\\w*|calculat\\w*|normaliz\\w*|convert\\w*|except|unless|instead|knowledge)\\b", | |
| re.IGNORECASE, | |
| ) | |
| _CELL = "\\(?[+-]?\\$?[\\d,]+\\.\\d{2}\\)?" | |
| _SCOPE_WORDS = { | |
| "only", | |
| "explicit", | |
| "printed", | |
| "separately", | |
| "tax", | |
| "taxes", | |
| "row", | |
| "rows", | |
| "fee", | |
| "fees", | |
| "billed", | |
| "goods", | |
| "product", | |
| "products", | |
| "services", | |
| "service", | |
| "charge", | |
| "charges", | |
| "exclude", | |
| "excluding", | |
| "all", | |
| "subtotal", | |
| "subtotals", | |
| "total", | |
| "totals", | |
| "grand", | |
| "payment", | |
| "payments", | |
| "amount", | |
| "due", | |
| "and", | |
| "or", | |
| "null", | |
| "if", | |
| "no", | |
| "when", | |
| "the", | |
| "document", | |
| "has", | |
| "a", | |
| "must", | |
| "be", | |
| "surcharge", | |
| "surcharges", | |
| "return", | |
| "instructions", | |
| } | |
| def role(label: str) -> str: | |
| label = " ".join(label.split()).rstrip(": ") | |
| if _SUMMARY.fullmatch(label): | |
| return "summary" | |
| if _PAYMENT.search(label): | |
| return "payment" | |
| if _TAX.search(label): | |
| return "tax" | |
| return "fee" if _FEE.search(label) else "charge" | |
| def scope(spec: dict[str, Any]) -> str | None: | |
| item = spec.get("items") | |
| if ( | |
| spec.get("type") != "list" | |
| or spec.get("knowledge") | |
| or (not isinstance(item, dict)) | |
| ): | |
| return None | |
| properties = item.get("properties", {}) | |
| if item.get("type") != "object" or len(properties) != 2: | |
| return None | |
| if any( | |
| ( | |
| not isinstance(child, dict) or child.get("knowledge") | |
| for child in properties.values() | |
| ) | |
| ): | |
| return None | |
| if sorted((child.get("type", "") for child in properties.values())) != [ | |
| "number", | |
| "text", | |
| ]: | |
| return None | |
| amount = next((child for child in properties.values() if child["type"] == "number")) | |
| if amount.get("decimals") != 2 or amount.get("unit") or amount.get("integer"): | |
| return None | |
| instructions = spec.get("instructions", "") | |
| if not isinstance(instructions, str) or _TRANSFORM.search(instructions): | |
| return None | |
| if set(re.findall("[a-z]+", instructions.lower())) - _SCOPE_WORDS: | |
| return None | |
| if any( | |
| ( | |
| set(child) - {"type", "instructions", "nullable", "decimals"} | |
| for child in properties.values() | |
| ) | |
| ): | |
| return None | |
| if any( | |
| ( | |
| _TRANSFORM.search(child.get("instructions", "")) | |
| for child in properties.values() | |
| ) | |
| ): | |
| return None | |
| positive = re.split( | |
| "[.\\n]|\\bExclude\\b", instructions, maxsplit=1, flags=re.IGNORECASE | |
| )[0].strip() | |
| if re.fullmatch( | |
| "Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?tax(?:es)?\\s+rows", | |
| positive, | |
| re.IGNORECASE, | |
| ): | |
| return "tax" | |
| if re.fullmatch( | |
| "Only\\s+(?:(?:explicit|printed|separately\\s+printed)\\s+)?fees?\\s+rows", | |
| positive, | |
| re.IGNORECASE, | |
| ): | |
| return "fee" | |
| if ( | |
| re.fullmatch( | |
| "Only\\s+billed\\s+(?:(?:goods?|products?|services?)\\s+(?:(?:and|or)\\s+)?)+(?:charge\\s+)?rows", | |
| positive, | |
| re.IGNORECASE, | |
| ) | |
| and all( | |
| ( | |
| re.search( | |
| "\\bExclude\\b[^.\\n]*\\b" + token + "\\b", | |
| instructions, | |
| re.IGNORECASE, | |
| ) | |
| for token in ("tax(?:es)?", "fees?", "payments?") | |
| ) | |
| ) | |
| and ( | |
| not re.search("\\b(?:not|never)\\s+exclude\\b", instructions, re.IGNORECASE) | |
| ) | |
| ): | |
| return "charge" | |
| return None | |
| def parse(line: object) -> tuple[str, int | float | None] | None: | |
| if not isinstance(line, str): | |
| raise TypeError("Missing literal image line") | |
| match = re.fullmatch("(.*?)\\s*(?:\\||:)\\s*(" + _CELL + ")\\s*\\|?\\s*", line) | |
| if not match: | |
| match = re.fullmatch("(.*?)\\s+(\\(?[+-]?\\$[\\d,]+\\.\\d{2}\\)?)\\s*", line) | |
| if not match: | |
| return None | |
| label = match[1].rstrip(" |:") | |
| if not label or "|" in label: | |
| return None | |
| return ( | |
| label, | |
| printed_numbers.normalize(match[2], {"type": "number", "decimals": 2}), | |
| ) | |
| def bands(image: Image.Image) -> list[tuple[int, int]] | None: | |
| if ( | |
| image.mode != "RGB" | |
| or image.width < 100 | |
| or image.getpixel((0, 0)) != (255, 255, 255) | |
| ): | |
| return None | |
| ink = np.asarray(image.convert("L")) < 200 | |
| if np.mean(ink) > 0.22: | |
| return None | |
| rows = image_table_rows._runs(ink.any(axis=1)) | |
| if not 2 <= len(rows) <= MAX_LIST_ITEMS or any( | |
| (not 8 <= bottom - top <= 32 for (top, bottom) in rows) | |
| ): | |
| return None | |
| return rows | |
| def prepare( | |
| request: PredictRequest, | |
| encoded: EncodedRequest, | |
| encode: Callable[..., EncodedRequest], | |
| ) -> EncodedRequest: | |
| (record, forecasts, tasks, typed, eos) = encoded | |
| annotations = record["_meta"]["extraction_contract"]["fields"] | |
| selected = { | |
| name: kind | |
| for (name, spec) in request.fields.items() | |
| if (kind := scope(spec)) and annotations.get(name, {}).get("query_fields") | |
| } | |
| if not selected or len(request.images) != 1 or request.documents: | |
| return encoded | |
| originals = record.get("_original_images") | |
| image = originals[0] if originals else request.images[0].to_pil() | |
| rows = bands(image) | |
| if rows is None: | |
| return encoded | |
| if ( | |
| image_table_rows.detect(image) is not None | |
| or image_ragged_rows.detect(image) is not None | |
| ): | |
| selected = {name: kind for (name, kind) in selected.items() if kind != "charge"} | |
| if not selected: | |
| return encoded | |
| salts = dict(record.get("_ar_task_salts", {})) | |
| def add(child_request: PredictRequest, **options: Any) -> list[str]: | |
| (child, _, prepared, _, _) = encode( | |
| child_request, physical_row_view=True, **options | |
| ) | |
| names = [] | |
| for source, ids, payload in prepared: | |
| target = "financial_line_" + str(len(record["fields"])) | |
| while target in record["fields"]: | |
| target += "_" | |
| record["fields"][target] = child["fields"][source] | |
| record["_ar_constraints"][target] = child["_ar_constraints"][source] | |
| salts[target] = child["_cache_salt"] | |
| tasks.append((target, ids, payload)) | |
| names.append(target) | |
| return names | |
| tasks = list(tasks) | |
| queries = [] | |
| for top, bottom in rows: | |
| view = image.crop( | |
| (0, max(0, top - 3), image.width, min(image.height, bottom + 3)) | |
| ) | |
| view = view.resize((view.width * 2, view.height * 2), Image.Resampling.BICUBIC) | |
| child_request = request.model_copy(update={"fields": {"printed_line": LINE}}) | |
| queries.extend( | |
| add( | |
| child_request, | |
| image_views=lambda view=view: (view,), | |
| image_view_key=f"financial-line-v1:{top}:{bottom}", | |
| ) | |
| ) | |
| plans = {} | |
| for name, kind in selected.items(): | |
| spec = request.fields[name] | |
| presence = { | |
| "type": "enum", | |
| "enum": [True, None], | |
| "nullable": True, | |
| "instructions": "Decide whether at least one printed row satisfies this requested list scope. " | |
| + spec.get("instructions", "") | |
| + " Return true if any matching printed row exists; return null only if there are no matching rows. A matching row with amount zero or a negative amount is present. Check the row's own label; a neighboring tax or fee does not remove an eligible service row. This asks only whether a matching row exists, not its value or row count.", | |
| } | |
| gate = add(request.model_copy(update={"fields": {name: presence}}))[0] | |
| previous = annotations[name].pop("query_fields") | |
| tasks = [task for task in tasks if task[0] not in {name, *previous}] | |
| for child in previous: | |
| record["fields"].pop(child) | |
| record["_ar_constraints"].pop(child, None) | |
| annotations.pop(child, None) | |
| plans[name] = {"kind": kind, "presence": gate, "spec": deepcopy(spec)} | |
| record.update( | |
| _financial_line_queries=queries, | |
| _financial_line_plans=plans, | |
| _ar_tasks=tasks, | |
| _ar_task_salts=salts, | |
| _ar_prefix=[], | |
| _ar_group_prefixes=[], | |
| ) | |
| return (record, forecasts, tasks, typed, eos) | |
| def assemble( | |
| record: dict[str, Any], predictions: dict[str, dict[str, Any]] | |
| ) -> dict[str, dict[str, Any]]: | |
| queries = record.get("_financial_line_queries", []) | |
| if not queries: | |
| return {} | |
| copies = [predictions[query] for query in queries] | |
| (parsed, parse_valid) = ([], True) | |
| for answer in copies: | |
| try: | |
| parsed.append( | |
| parse(answer["value"]) if isinstance(answer.get("value"), str) else None | |
| ) | |
| except (ValueError, TypeError, OverflowError): | |
| parsed.append(None) | |
| parse_valid = False | |
| result = {} | |
| for name, plan in record["_financial_line_plans"].items(): | |
| spec = plan["spec"] | |
| properties = spec["items"]["properties"] | |
| text = next( | |
| (key for (key, value) in properties.items() if value["type"] == "text") | |
| ) | |
| number = next( | |
| (key for (key, value) in properties.items() if value["type"] == "number") | |
| ) | |
| selected = [ | |
| (answer, row) | |
| for (answer, row) in zip(copies, parsed, strict=True) | |
| if row is not None and role(row[0]) == plan["kind"] | |
| ] | |
| values = [{text: row[0], number: row[1]} for (_, row) in selected] | |
| presence = predictions[plan["presence"]] | |
| relevant = [*copies, presence] | |
| complete = all((answer.get("complete", False) for answer in relevant)) | |
| valid = ( | |
| parse_valid | |
| and complete | |
| and all((answer.get("valid", False) for answer in relevant)) | |
| ) | |
| valid &= len(values) <= spec.get("max_items", MAX_LIST_ITEMS) | |
| try: | |
| for value in values: | |
| Draft202012Validator(ar.schema(spec["items"])).validate(value) | |
| except ValidationError: | |
| valid = False | |
| p_null = presence["p_null"] | |
| if ( | |
| type(p_null) not in (int, float) | |
| or not math.isfinite(p_null) | |
| or (not 0 <= p_null <= 1) | |
| or any( | |
| ( | |
| type(answer.get("log_probability")) not in (int, float) | |
| or not math.isfinite(answer["log_probability"]) | |
| or answer["log_probability"] > 0 | |
| for answer in copies | |
| ) | |
| ) | |
| ): | |
| raise ValueError("Invalid financial-line model scores") | |
| event = 1 - p_null if values else p_null | |
| probability = ( | |
| sum((answer["log_probability"] for answer in copies)) + math.log(event) | |
| if event > 0 | |
| else None | |
| ) | |
| answer = { | |
| "value": values, | |
| "valid": bool(valid), | |
| "complete": complete, | |
| "confidence": 0.0, | |
| "confidence_method": "ar_literal_image_lines_and_scope_presence_probability", | |
| "confidences": [copy["confidence"] for (copy, _) in selected], | |
| "p_null": p_null, | |
| "p_null_method": "ar_parent_scope_presence_null_probability", | |
| "p_nulls": [copy["p_null"] for (copy, _) in selected], | |
| "spans": [None] * len(selected), | |
| } | |
| if probability is not None: | |
| answer.update( | |
| log_probability=probability, | |
| confidence=math.exp(probability) if valid else 0.0, | |
| ) | |
| result[name] = answer | |
| return result | |