Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic-flash with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic-flash with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic-flash") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic-flash") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic-flash", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic-flash with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic-flash" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic-flash
- SGLang
How to use Cloudflare/clairic-flash with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic-flash with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic-flash
File size: 14,427 Bytes
24191a8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 | """Generic startup GDN decoder graphs with owned native state prefixes."""
from __future__ import annotations
from contextlib import contextmanager
import os
import threading
import time
from types import MethodType
from typing import Any, Iterator
import weakref
import torch
from .decoder_capture import DecoderGraph, TorchBackend, callable_key
from .gdn_pool import NativeStatePool
class _SelectedCache:
"""Expose one real native layer to the proved single-layer capture adapter."""
def __init__(self, cache: Any, index: int) -> None:
self.cache, self.index = cache, index
self.layers = [cache.layers[index]]
def has_previous_state(self, index: int) -> bool:
assert index == 0
return self.cache.has_previous_state(self.index)
def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor:
assert index == 0
return self.cache.update_recurrent_state(state, self.index)
class DecoderGraphs:
"""One engine, CUDA stream and worker; native prefill and dense attention stay eager."""
def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None:
self.backbone = backbone
self.thread = threading.get_ident()
self.device = backbone.device
self.stream = torch.cuda.current_stream(self.device)
self.max_batch, self.buckets = max_batch, buckets
self.pool = NativeStatePool(backbone, max_batch)
self.graphs: dict[tuple[int, int], DecoderGraph] = {}
self.wrappers: list[tuple[Any, Any, Any]] = []
self.active: tuple[Any, int] | None = None
self.closed = False
self.preparing = True
self.stats = {"replay": 0, "eager": 0, "retired": 0}
self.cleanup: list[dict[str, Any]] = []
self.preparation: list[dict[str, Any]] = []
self.observe_requests = False
self.last_request: dict[str, Any] | None = None
self.active_methods: dict[int, Any] = {}
def _owner(self) -> bool:
return (not self.closed and threading.get_ident() == self.thread
and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream)
@staticmethod
def _runtime() -> tuple[Any, ...]:
import deep_gemm as dg
return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(),
tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_"))))
def prepare(self) -> None:
if not self._owner():
raise RuntimeError("Graph preparation must use the engine CUDA worker")
backend = TorchBackend(torch, self._runtime)
try:
for index in self.pool.states:
layer = self.pool.language.layers[index]
eager = layer.forward
weak_manager = weakref.ref(self)
def forward(this: Any, *args: Any, _index: int = index,
_eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any:
manager = _manager()
if manager is None:
return _eager(*args, **kwargs)
return manager._forward(_index, _eager, args, kwargs)
installed = MethodType(forward, layer)
self.wrappers.append((layer, eager, installed))
layer.forward = installed
dtype = torch.bfloat16 if index == 0 else torch.float32
for bucket in self.buckets:
began = time.perf_counter_ns()
dummy = self.pool.new_cache(bucket, previous=True)
selected = _SelectedCache(dummy, index)
def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any:
return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None)
zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(),
dtype=dtype, device=self.device)
self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero,
operation, exclusive_process=True)
entry = self.graphs[index, bucket].entry
self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype),
"capture_ms": (time.perf_counter_ns() - began) / 1e6,
"generic_zero_input": True,
"capture_mutation_restored": entry["capture_mutation_restored"],
"pool": list(entry["pool"]),
"pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"],
"static_input_storage_bytes": entry["static_input_storage_bytes"]})
self.pool.zero()
self.stream.synchronize()
self.preparing = False
except BaseException:
self.close()
raise
@contextmanager
def request(self, batch: int) -> Iterator[Any]:
if (not self._owner() or self.preparing or self.active is not None
or not 1 <= batch <= self.max_batch or torch.is_grad_enabled()
or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()):
yield None
return
cache = self.pool.new_cache(batch)
if not self.pool.owns(cache, batch):
raise RuntimeError("Fresh cache must use owned native state prefixes")
self.active = (cache, batch)
self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states}
completed = False
try:
yield cache
completed = True
finally:
try:
if self.observe_requests:
self.last_request = {"batch": batch, "completed": completed,
"cache_type": type(cache).__name__,
"native_prefixes_owned": self.pool.owns(cache, batch),
"layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape),
"conv_stride": list(cache.layers[index].conv_states.stride()),
"conv_dtype": str(cache.layers[index].conv_states.dtype),
"recurrent_shape": list(cache.layers[index].recurrent_states.shape),
"recurrent_stride": list(cache.layers[index].recurrent_states.stride()),
"recurrent_dtype": str(cache.layers[index].recurrent_states.dtype),
"has_previous_state": bool(cache.layers[index].has_previous_state)}
for index in self.pool.states]}
finally:
self.active = None
self.active_methods.clear()
@staticmethod
def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]:
layer = cache.layers[index]
return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state),
callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state),
layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size)
def diagnostics(self) -> dict[str, Any]:
"""Primitive-only snapshot for separate untimed diagnostics and shutdown."""
return {"closed": self.closed, "preparing": self.preparing,
"active": self.active is not None, "owner_thread": self.thread,
"stream": self.stream.cuda_stream, "max_batch": self.max_batch,
"buckets": list(self.buckets), "graphs": len(self.graphs),
"arena_storage_bytes": self.pool.storage_bytes,
"stats": dict(self.stats), "preparation": list(self.preparation),
"observe_requests": self.observe_requests,
"last_request": self.last_request, "cleanup": list(self.cleanup)}
def _retire(self, index: int) -> None:
for key in tuple(self.graphs):
if key[0] == index:
graph = self.graphs[key]
details = graph.close()
del self.graphs[key]
self.cleanup.append({"layer": key[0], "bucket": key[1], **details})
if not details["passed"]:
raise RuntimeError("Decoder graph retirement failed strict cleanup")
self.stats["retired"] += 1
def _owns_layer(self, cache: Any, batch: int, index: int) -> bool:
layer = cache.layers[index]
for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True):
if (actual is None or actual.data_ptr() != owned.data_ptr()
or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
or actual.dtype != owned.dtype or actual.device != owned.device
or actual.storage_offset() != owned.storage_offset()
or tuple(actual.shape) != (batch, *owned.shape[1:])
or tuple(actual.stride()) != tuple(owned.stride())):
return False
return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized
def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any:
value = args[0] if args else kwargs.get("hidden_states")
cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None)
mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None)
batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0
expected_dtype = torch.bfloat16 if index == 0 else torch.float32
admitted = (not self.preparing and self._owner() and self.active is not None
and cache is self.active[0] and batch == self.active[1]
and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype
and value.device == self.device and mask is None and not torch.is_grad_enabled()
and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids",
"past_key_values", "use_cache", "cache_position"}
and self._cache_methods(cache, index) == self.active_methods.get(index)
and self._owns_layer(cache, batch, index) and cache.has_previous_state(index))
if not admitted:
self.stats["eager"] += 1
return eager(*args, **kwargs)
bucket = next((bound for bound in self.buckets if batch <= bound), None)
graph = self.graphs.get((index, bucket))
if graph is None:
self.stats["eager"] += 1
return eager(*args, **kwargs)
entry = graph.entry
try:
graph.check(entry["static_input"])
except RuntimeError:
self._retire(index)
self.stats["eager"] += 1
return eager(*args, **kwargs)
entry["static_input"][:batch].copy_(value)
if batch < bucket:
entry["static_input"][batch:].zero_()
entry["graph"].replay()
self.stats["replay"] += 1
return entry["static_output"][:batch].clone()
def close(self) -> None:
if self.closed:
return
if not self._owner():
raise RuntimeError("Graph cleanup must use the engine CUDA worker")
self.active = None
self.active_methods.clear()
error = None
try:
for index in tuple(self.pool.states):
try:
self._retire(index)
except BaseException as exc:
if error is None:
error = exc
finally:
for layer, original, installed in reversed(self.wrappers):
if layer.forward is installed:
layer.forward = original
self.wrappers.clear()
self.pool.close()
self.backbone = None
self.closed = True
if error is not None:
raise error
def _admissible(backbone: Any, max_batch: int) -> bool:
"""Reject unsupported contexts before allocating arenas or installing wrappers."""
config = backbone.config.get_text_config()
limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0
if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16
or not 1 <= max_batch <= limit):
return False
registry = torch.nn.modules.module
if registry._global_forward_hooks or registry._global_forward_pre_hooks:
return False
for module in backbone.modules():
if module._forward_hooks or module._forward_pre_hooks:
return False
for layer in backbone.model.language_model.layers:
if layer.layer_type != "linear_attention":
continue
for value in layer.parameters():
try:
version = value._version
except RuntimeError:
return False
if version is None:
return False
return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None:
"""Run on the existing worker after math installation; unsupported engines stay eager."""
if not _admissible(backbone, max_batch):
return None
largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound)
buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest)
with torch.no_grad():
manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets)
manager.prepare()
return manager
|