runtime.py

"""FrameLite inference on ZeroGPU, using the released LoRA and vision merger."""
from __future__ import annotations

import os
from pathlib import Path
from typing import Any

# Import before torch or transformers so ZeroGPU can prepare CUDA at startup.
import spaces
from huggingface_hub import snapshot_download
from PIL import Image

from framelite import FrameLite

ROOT = Path(__file__).resolve().parent
MODEL_REPO = "zlab-sjtu/FrameLite"
BASE_REPO = "Qwen/Qwen3.5-0.8B"


def _load_model() -> FrameLite:
    directory = Path(os.environ.get("FRAMELITE_WEIGHTS", ROOT / "weights/framelite")).expanduser()
    required = ["framelite.safetensors", "inference_config.json"]
    if not all((directory / name).is_file() for name in required):
        snapshot_download(
            MODEL_REPO,
            local_dir=str(directory),
            allow_patterns=required + ["LICENSE", "NOTICE"],
            token=False,
        )

    base = directory / "base"
    if not (
        (base / "config.json").is_file()
        and (base / "tokenizer.json").is_file()
        and any(base.glob("*.safetensors"))
    ):
        snapshot_download(
            BASE_REPO,
            local_dir=str(base),
            allow_patterns=["*.json", "*.safetensors", "*.txt", "*.jinja", "LICENSE"],
            token=False,
        )

    # An explicit CPU setting is available for local use; the hosted demo uses BF16 CUDA.
    device = os.environ.get("FRAMELITE_DEVICE", "cuda:0")
    dtype = "float32" if device == "cpu" else "bfloat16"
    return FrameLite.from_pretrained(directory, device=device, dtype=dtype)


# ZeroGPU optimizes model transfers when CUDA placement happens during startup.
MODEL = _load_model()


@spaces.GPU(duration=30)
def predict(image: Image.Image) -> dict[str, Any]:
    """Return one greedy crop with the released prompt and preprocessing settings."""
    if not isinstance(image, Image.Image):
        raise TypeError("Please upload an image first.")
    return MODEL.predict(image)