lucataco's picture
Clairic Flash, MLX 4-bit, with runtime, server and conversion script
ae97c5d verified
Raw History Blame Contribute Delete
3.51 kB
"""A torch-free stand-in for the Transformers `Qwen3VLProcessor` used by the Clairic request pipeline.
Not part of the Cloudflare release. The upstream pipeline calls four things on its processor: the chat
template, `replace_image_token`, `get_text_with_replacements`, and `processor(text=..., images=...)`. This
class provides exactly those on top of the tokenizer and mlx-vlm's NumPy image processor, so Transformers'
torchvision-based image and video processors are not needed.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import numpy as np
MIN_PIXELS = 65_536
class ClairicProcessor:
image_token = "<|image_pad|>"
def __init__(self, path: Path, tokenizer: Any, max_pixels: int) -> None:
from mlx_vlm.models.qwen3_vl.processing_qwen3_vl import Qwen3VLImageProcessor
config = json.loads((path / "preprocessor_config.json").read_text())
self.tokenizer = tokenizer
self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)
self.image_processor = Qwen3VLImageProcessor(
patch_size=config["patch_size"], temporal_patch_size=config["temporal_patch_size"],
merge_size=config["merge_size"], min_pixels=config["size"].get("shortest_edge", MIN_PIXELS),
max_pixels=max_pixels, image_mean=config["image_mean"], image_std=config["image_std"])
def apply_chat_template(self, messages: list[dict[str, Any]], **kwargs: Any) -> Any:
return self.tokenizer.apply_chat_template(messages, **kwargs)
def replace_image_token(self, image_inputs: dict[str, Any], image_idx: int) -> str:
merge = self.image_processor.merge_size ** 2
return self.image_token * (int(np.prod(np.asarray(image_inputs["image_grid_thw"])[image_idx])) // merge)
def get_text_with_replacements(self, text: list[str], images_replacements: list[str] = (),
videos_replacements: list[str] = (), audio_replacements: list[str] = ()) -> tuple[list[str], list]:
"""Replace each image placeholder, in order across the batch, with its expanded run of placeholders."""
if videos_replacements or audio_replacements:
raise ValueError("Clairic accepts images only")
replacements = iter(images_replacements)
expanded = []
for sample in text:
pieces = sample.split(self.image_token)
if len(images_replacements) == 0:
expanded.append(sample)
continue
expanded.append("".join(piece + (next(replacements) if index < len(pieces) - 1 else "")
for index, piece in enumerate(pieces)))
return expanded, []
def __call__(self, text: list[str], images: list[Any], return_tensors: str | None = None) -> dict[str, Any]:
processed = self.image_processor(list(images))
media = {"pixel_values": np.asarray(processed["pixel_values"], dtype=np.float32),
"image_grid_thw": np.asarray(processed["image_grid_thw"], dtype=np.int64)}
expanded, _ = self.get_text_with_replacements(
list(text), [self.replace_image_token(media, index) for index in range(len(images))])
ids = np.asarray(self.tokenizer(expanded, add_special_tokens=False)["input_ids"], dtype=np.int64)
return {"input_ids": ids, "attention_mask": np.ones_like(ids), **media,
"mm_token_type_ids": (ids == self.image_token_id).astype(np.int64)}