Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic-flash with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic-flash with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic-flash") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic-flash") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic-flash", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic-flash with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic-flash" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic-flash
- SGLang
How to use Cloudflare/clairic-flash with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic-flash with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic-flash
Download clairic_runtime/po_image_supplement.py from Cloudflare/clairic-flash: direct link, hf CLI and curl.
- Browser
- Download file 8.2 kB
-
https://huggingface.co/Cloudflare/clairic-flash/resolve/main/clairic_runtime/po_image_supplement.py
- Command line
-
hf download hf://Cloudflare/clairic-flash/clairic_runtime/po_image_supplement.py
-
curl -L -o po_image_supplement.py https://huggingface.co/Cloudflare/clairic-flash/resolve/main/clairic_runtime/po_image_supplement.py
8.2 kB
| from __future__ import annotations | |
| import base64 | |
| import io | |
| import json | |
| from typing import TYPE_CHECKING | |
| from .api import ClairImage | |
| from . import ar, grounding, identifier_spelling, image_focus | |
| from PIL import Image | |
| if TYPE_CHECKING: | |
| from collections.abc import Callable, Sequence | |
| from typing import Any | |
| from .api import PredictRequest | |
| from .backend import ARTask, EncodedRequest | |
| from transformers import PreTrainedTokenizerBase | |
| VERSION = "27b-po-all-content-horizontal2-supplement-v2" | |
| MAX_PIXELS = 1600000 | |
| def supplement(images: Sequence[Image.Image]) -> ClairImage: | |
| views: list[Image.Image] = [] | |
| for image in images: | |
| focused = image_focus.focus_document(image) | |
| views.append( | |
| focused.resize( | |
| (round(focused.width * 2.0), focused.height), Image.Resampling.BICUBIC | |
| ) | |
| ) | |
| board = Image.new( | |
| "RGB", | |
| ( | |
| max((image.width for image in views)), | |
| sum((image.height for image in views)) + 32 * (len(views) - 1), | |
| ), | |
| "white", | |
| ) | |
| y = 0 | |
| for image in views: | |
| board.paste(image, (0, y)) | |
| y += image.height + 32 | |
| scale = min(1.0, (MAX_PIXELS / (board.width * board.height)) ** 0.5) | |
| if scale < 1: | |
| board = board.resize( | |
| (max(1, int(board.width * scale)), max(1, int(board.height * scale))), | |
| Image.Resampling.BICUBIC, | |
| ) | |
| buffer = io.BytesIO() | |
| board.save(buffer, format="PNG") | |
| return ClairImage.model_validate( | |
| { | |
| "content_type": "image/png", | |
| "base64": base64.b64encode(buffer.getvalue()).decode(), | |
| } | |
| ) | |
| def plain_task( | |
| task: ARTask, | |
| wire_spec: dict[str, Any], | |
| public_spec: dict[str, Any], | |
| tokenizer: PreTrainedTokenizerBase, | |
| max_length: int, | |
| ) -> tuple[ARTask, dict[str, Any], dict[str, str]]: | |
| (name, ids, media) = task | |
| source_ids = list(ids) | |
| text = tokenizer.decode( | |
| ids, skip_special_tokens=False, clean_up_tokenization_spaces=False | |
| ) | |
| if tokenizer.encode(text, add_special_tokens=False) != source_ids: | |
| raise ValueError( | |
| "PO scalar source prompt cannot make a lossless tokenizer roundtrip" | |
| ) | |
| before = identifier_spelling.INSTRUCTION + "\n" | |
| if text.count(before) != 1 or wire_spec.get("instructions", "").count(before) != 1: | |
| raise ValueError( | |
| "PO scalar source must contain exactly one spelling instruction" | |
| ) | |
| scalar = { | |
| **wire_spec, | |
| "instructions": wire_spec["instructions"].replace(before, ""), | |
| } | |
| for key in ("minLength", "maxLength", "pattern"): | |
| if key in public_spec: | |
| scalar[key] = public_spec[key] | |
| (old_schema, schema) = (ar.schema(wire_spec), ar.schema(scalar)) | |
| text = text.replace(before, "") | |
| if schema != old_schema: | |
| old = "\nSchema: " + json.dumps( | |
| old_schema, ensure_ascii=False, separators=(",", ":") | |
| ) | |
| new = "\nSchema: " + json.dumps( | |
| schema, ensure_ascii=False, separators=(",", ":") | |
| ) | |
| if text.count(old) != 1: | |
| raise ValueError("PO scalar source must contain exactly one native schema") | |
| text = text.replace(old, new) | |
| plain_ids = tokenizer.encode(text, add_special_tokens=False) | |
| if len(plain_ids) > max_length: | |
| raise ValueError("PO scalar detail exceeds the complete model context") | |
| return ((name, plain_ids, media), scalar, {"json_schema": json.dumps(schema)}) | |
| def selected_fields( | |
| request: PredictRequest, encoded: EncodedRequest | |
| ) -> dict[str, dict[str, Any]]: | |
| image_count = len(request.images) | |
| if image_count not in (1, 2, 3) or request.documents: | |
| return {} | |
| (record, _, tasks, _, _) = encoded | |
| contracts = record.get("_meta", {}).get("identifier_spelling", {}) | |
| active = {name for (name, _, media) in tasks if media is not None} | |
| return { | |
| name: request.fields[name] | |
| for (name, contract) in contracts.items() | |
| if name in active | |
| and contract.get("wire") == identifier_spelling.VERSION | |
| and (not request.fields[name].get("knowledge")) | |
| and (not grounding.state_only(request.fields[name])) | |
| } | |
| def prepare( | |
| request: PredictRequest, | |
| encoded: EncodedRequest, | |
| *, | |
| encode_child: Callable[[PredictRequest], EncodedRequest], | |
| tokenizer: PreTrainedTokenizerBase, | |
| max_length: int, | |
| media_groups: Callable[..., list[dict[str, Any]]], | |
| ) -> EncodedRequest: | |
| selected = selected_fields(request, encoded) | |
| if not selected: | |
| return encoded | |
| image_count = len(request.images) | |
| (record, forecasts, tasks, typed, eos) = encoded | |
| contracts = record["_meta"]["identifier_spelling"] | |
| originals = record.get("_original_images", ()) | |
| if len(originals) != image_count: | |
| raise ValueError("PO supplement must retain every original image in order") | |
| fields = image_focus.preserve_scalar_fields(selected) | |
| supplements = ( | |
| tuple((supplement([image]) for image in originals)) | |
| if image_count == 2 | |
| else (supplement(originals),) | |
| ) | |
| child_request = request.model_copy( | |
| update={"fields": fields, "images": (*request.images, *supplements)} | |
| ) | |
| (child, _, replacements, child_typed, child_eos) = encode_child(child_request) | |
| by_name = {task[0]: task for task in replacements} | |
| if ( | |
| child_typed is not None | |
| or child_eos != eos | |
| or set(by_name) != set(selected) | |
| or (len(replacements) != len(selected)) | |
| ): | |
| raise ValueError( | |
| "PO supplement must retain the complete selected scalar task contract" | |
| ) | |
| plain = image_count == 1 | |
| constraints: dict[str, dict[str, Any]] | |
| remaining: dict[str, Any] | list[str] | |
| (fields, constraints) = ({}, {}) | |
| for name, public_spec in selected.items(): | |
| if plain: | |
| (by_name[name], fields[name], constraints[name]) = plain_task( | |
| by_name[name], child["fields"][name], public_spec, tokenizer, max_length | |
| ) | |
| else: | |
| fields[name] = child["fields"][name] | |
| constraints[name] = child["_ar_constraints"][name] | |
| updated = [by_name.get(task[0], task) for task in tasks] | |
| salts = dict(record.get("_ar_task_salts", {})) | |
| for name in selected: | |
| record["fields"][name] = fields[name] | |
| record["_ar_constraints"][name] = constraints[name] | |
| salts[name] = child.get("_ar_task_salts", {}).get(name, child["_cache_salt"]) | |
| if plain: | |
| remaining = { | |
| name: contract | |
| for (name, contract) in contracts.items() | |
| if name not in selected | |
| } | |
| if remaining: | |
| record["_meta"]["identifier_spelling"] = remaining | |
| else: | |
| record["_meta"].pop("identifier_spelling", None) | |
| for key in ("model_document_scalar_view", "identifier_spelling_image_view"): | |
| previous = record["_meta"].get(key) | |
| if previous is None: | |
| continue | |
| remaining = [name for name in previous["fields"] if name not in selected] | |
| if not remaining: | |
| record["_meta"].pop(key) | |
| else: | |
| retained = {**previous, "fields": remaining} | |
| if "po_fields" in retained: | |
| retained["po_fields"] = [ | |
| name for name in retained["po_fields"] if name not in selected | |
| ] | |
| record["_meta"][key] = retained | |
| record["_meta"]["model_po_supplement"] = { | |
| "version": VERSION, | |
| "fields": list(selected), | |
| "original_image_count": image_count, | |
| "native_image_count": len(child_request.images), | |
| "horizontal_scale": 2.0, | |
| "supplement_max_pixels": MAX_PIXELS, | |
| "wire": "native_scalar" if plain else identifier_spelling.VERSION, | |
| } | |
| record.update( | |
| _ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[] | |
| ) | |
| record["_ar_media_groups"] = media_groups( | |
| updated, | |
| default_salt=record["_cache_salt"], | |
| task_salts=salts, | |
| annotations=record["_meta"]["extraction_contract"]["fields"], | |
| ) | |
| return (record, forecasts, updated, typed, eos) | |