File size: 1,998 Bytes
b2f3cdc d816847 b2f3cdc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | """FrameLite inference on ZeroGPU, using the released LoRA and vision merger."""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any
# Import before torch or transformers so ZeroGPU can prepare CUDA at startup.
import spaces
from huggingface_hub import snapshot_download
from PIL import Image
from framelite import FrameLite
ROOT = Path(__file__).resolve().parent
MODEL_REPO = "zlab-sjtu/FrameLite"
BASE_REPO = "Qwen/Qwen3.5-0.8B"
def _load_model() -> FrameLite:
directory = Path(os.environ.get("FRAMELITE_WEIGHTS", ROOT / "weights/framelite")).expanduser()
required = ["framelite.safetensors", "inference_config.json"]
if not all((directory / name).is_file() for name in required):
snapshot_download(
MODEL_REPO,
local_dir=str(directory),
allow_patterns=required + ["LICENSE", "NOTICE"],
token=False,
)
base = directory / "base"
if not (
(base / "config.json").is_file()
and (base / "tokenizer.json").is_file()
and any(base.glob("*.safetensors"))
):
snapshot_download(
BASE_REPO,
local_dir=str(base),
allow_patterns=["*.json", "*.safetensors", "*.txt", "*.jinja", "LICENSE"],
token=False,
)
# An explicit CPU setting is available for local use; the hosted demo uses BF16 CUDA.
device = os.environ.get("FRAMELITE_DEVICE", "cuda:0")
dtype = "float32" if device == "cpu" else "bfloat16"
return FrameLite.from_pretrained(directory, device=device, dtype=dtype)
# ZeroGPU optimizes model transfers when CUDA placement happens during startup.
MODEL = _load_model()
@spaces.GPU(duration=30)
def predict(image: Image.Image) -> dict[str, Any]:
"""Return one greedy crop with the released prompt and preprocessing settings."""
if not isinstance(image, Image.Image):
raise TypeError("Please upload an image first.")
return MODEL.predict(image)
|