Merge pull request #2 from MSghais/feat/poc_gradio

.env.example

@@ -1,12 +1,25 @@
+# --- Preset selection (models.yaml is the source of truth) ---
+ACTIVE_MODEL=qwen3b-gguf
+# Dev: enable dropdown in Gradio. Space: leave false to pin one model for visitors.
+ALLOW_MODEL_SWITCH=true
+# MODEL_PRESETS_PATH=./models.yaml
+
+# --- Legacy single-model overrides (optional; applied to ACTIVE_MODEL only) ---
 INFERENCE_BACKEND=llama_cpp
 MODEL_REPO=Qwen/Qwen2.5-3B-Instruct-GGUF
 MODEL_FILE=qwen2.5-3b-instruct-q4_k_m.gguf
 N_CTX=4096
 N_GPU_LAYERS=0
 
-# Optional: local GGUF path instead of Hub download
+# Optional: local GGUF path instead of Hub download (set in models.yaml model_path too)
 # MODEL_PATH=./models/qwen2.5-3b-instruct-q4_k_m.gguf
 
-# Optional: transformers backend (requires inference[transformers] extra)
+# Optional: transformers presets (requires inference[transformers] extra)
+# ACTIVE_MODEL=minicpm5-1b
 # INFERENCE_BACKEND=transformers
-# MODEL_ID=Qwen/Qwen2.5-3B-Instruct
+# MODEL_ID=openbmb/MiniCPM5-1B
+# TRUST_REMOTE_CODE=true
+
+# Optional: local fine-tuned merged weights
+# ACTIVE_MODEL=gemma-merged-local
+# MODEL_ID=./gemma_merged_model

Dockerfile

@@ -13,7 +13,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 WORKDIR /app
 
-COPY pyproject.toml uv.lock .python-version README.md ./
+COPY pyproject.toml uv.lock .python-version README.md models.yaml ./
 COPY apps/gradio-space/pyproject.toml apps/gradio-space/README.md apps/gradio-space/
 COPY libs/inference/pyproject.toml libs/inference/README.md libs/inference/
 COPY apps/gradio-space/src apps/gradio-space/src

README.md

@@ -1,5 +1,6 @@
 ---
-title: Small Model Hackathon
+
+## title: Small Model Hackathon
 emoji: 🦙
 colorFrom: blue
 colorTo: green
@@ -7,7 +8,6 @@ sdk: docker
 app_port: 7860
 pinned: false
 license: apache-2.0
----
 
 # Small Model Hackathon
 
@@ -33,21 +33,23 @@ uv run python scripts/download_model.py
 uv run --package gradio-space python -m gradio_space.app
 ```
 
-Open http://localhost:7860. The model downloads from Hugging Face Hub on the first chat message (or set `MODEL_PATH` to a local GGUF).
+Open [http://localhost:7860](http://localhost:7860). The model downloads from Hugging Face Hub on the first chat message (or set `MODEL_PATH` to a local GGUF).
 
 ## Environment variables
 
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `INFERENCE_BACKEND` | `llama_cpp` | `llama_cpp` or `transformers` |
-| `MODEL_REPO` | `Qwen/Qwen2.5-3B-Instruct-GGUF` | Hub repo for GGUF |
-| `MODEL_FILE` | `qwen2.5-3b-instruct-q4_k_m.gguf` | GGUF filename |
-| `MODEL_PATH` | — | Local GGUF path (skips Hub download) |
-| `N_CTX` | `4096` | Context window |
-| `N_GPU_LAYERS` | `0` | GPU layers for llama.cpp (0 = CPU) |
-| `MODEL_ID` | `Qwen/Qwen2.5-3B-Instruct` | Used when `INFERENCE_BACKEND=transformers` |
 
-See [`.env.example`](.env.example) for a full template.
+| Variable            | Default                           | Description                                |
+| ------------------- | --------------------------------- | ------------------------------------------ |
+| `INFERENCE_BACKEND` | `llama_cpp`                       | `llama_cpp` or `transformers`              |
+| `MODEL_REPO`        | `Qwen/Qwen2.5-3B-Instruct-GGUF`   | Hub repo for GGUF                          |
+| `MODEL_FILE`        | `qwen2.5-3b-instruct-q4_k_m.gguf` | GGUF filename                              |
+| `MODEL_PATH`        | —                                 | Local GGUF path (skips Hub download)       |
+| `N_CTX`             | `4096`                            | Context window                             |
+| `N_GPU_LAYERS`      | `0`                               | GPU layers for llama.cpp (0 = CPU)         |
+| `MODEL_ID`          | `Qwen/Qwen2.5-3B-Instruct`        | Used when `INFERENCE_BACKEND=transformers` |
+
+
+See `[.env.example](.env.example)` for a full template.
 
 ## Monorepo layout
 
@@ -81,11 +83,11 @@ docker run --rm -p 7860:7860 -e MODEL_REPO=Qwen/Qwen2.5-3B-Instruct-GGUF hackath
 
 ## Hackathon checklist
 
-- [ ] Choose a track (Backyard AI or Thousand Token Wood)
-- [ ] Space live under build-small-hackathon
-- [ ] Demo video recorded
-- [ ] Social post published
-- [ ] Submission locked in by **June 15, 2026**
+- Choose a track (Backyard AI or Thousand Token Wood)
+- Space live under build-small-hackathon
+- Demo video recorded
+- Social post published
+- Submission locked in by **June 15, 2026**
 
 ### Badge targets
 
@@ -101,3 +103,4 @@ uv sync --package inference --extra transformers
 INFERENCE_BACKEND=transformers MODEL_ID=Qwen/Qwen2.5-3B-Instruct \
   uv run --package gradio-space python -m gradio_space.app
 ```
+

USAGE.md

@@ -45,7 +45,7 @@ MODEL_PATH=./models/qwen2.5-3b-instruct-q4_k_m.gguf
 uv run --package gradio-space python -m gradio_space.app
 ```
 
-Open http://localhost:7860.
+Open [http://localhost:7860](http://localhost:7860).
 
 The model loads on the **first chat message** unless you set `MODEL_PATH`. After code changes, restart the process to pick up updates.
 
@@ -61,16 +61,18 @@ uv run --package gradio-space python -c "from gradio_space.app import build_demo
 
 ### Local env reference
 
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `INFERENCE_BACKEND` | `llama_cpp` | `llama_cpp` or `transformers` |
-| `MODEL_REPO` | `Qwen/Qwen2.5-3B-Instruct-GGUF` | Hub repo for GGUF |
-| `MODEL_FILE` | `qwen2.5-3b-instruct-q4_k_m.gguf` | GGUF filename |
-| `MODEL_PATH` | — | Local GGUF path (skips Hub download) |
-| `N_CTX` | `4096` | Context window |
-| `N_GPU_LAYERS` | `0` | GPU layers for llama.cpp (`0` = CPU only) |
-| `PORT` | `7860` | Gradio listen port |
-| `MODEL_ID` | `Qwen/Qwen2.5-3B-Instruct` | Used when `INFERENCE_BACKEND=transformers` |
+
+| Variable            | Default                           | Description                                |
+| ------------------- | --------------------------------- | ------------------------------------------ |
+| `INFERENCE_BACKEND` | `llama_cpp`                       | `llama_cpp` or `transformers`              |
+| `MODEL_REPO`        | `Qwen/Qwen2.5-3B-Instruct-GGUF`   | Hub repo for GGUF                          |
+| `MODEL_FILE`        | `qwen2.5-3b-instruct-q4_k_m.gguf` | GGUF filename                              |
+| `MODEL_PATH`        | —                                 | Local GGUF path (skips Hub download)       |
+| `N_CTX`             | `4096`                            | Context window                             |
+| `N_GPU_LAYERS`      | `0`                               | GPU layers for llama.cpp (`0` = CPU only)  |
+| `PORT`              | `7860`                            | Gradio listen port                         |
+| `MODEL_ID`          | `Qwen/Qwen2.5-3B-Instruct`        | Used when `INFERENCE_BACKEND=transformers` |
+
 
 ### Optional: transformers backend
 
@@ -98,7 +100,7 @@ docker run --rm -p 7860:7860 \
   hackathon-space
 ```
 
-Open http://localhost:7860. Stop with `Ctrl+C`.
+Open [http://localhost:7860](http://localhost:7860). Stop with `Ctrl+C`.
 
 To use a pre-downloaded local model inside Docker, mount it and set `MODEL_PATH`:
 
@@ -142,22 +144,26 @@ hf repo create build-small-hackathon/<your-space-name> \
 
 ### 3. Configure hardware
 
-| Setting | Recommendation |
-|---------|----------------|
-| Hardware | **CPU basic** to start (llama.cpp with `N_GPU_LAYERS=0`) |
-| Upgrade | GPU Space if you set `N_GPU_LAYERS > 0` for faster inference |
+
+| Setting  | Recommendation                                               |
+| -------- | ------------------------------------------------------------ |
+| Hardware | **CPU basic** to start (llama.cpp with `N_GPU_LAYERS=0`)     |
+| Upgrade  | GPU Space if you set `N_GPU_LAYERS > 0` for faster inference |
+
 
 ### 4. Set Space environment variables
 
 In the Space **Settings → Variables and secrets**:
 
-| Variable | Value |
-|----------|-------|
-| `INFERENCE_BACKEND` | `llama_cpp` |
-| `MODEL_REPO` | `Qwen/Qwen2.5-3B-Instruct-GGUF` |
-| `MODEL_FILE` | `qwen2.5-3b-instruct-q4_k_m.gguf` |
-| `N_CTX` | `4096` |
-| `N_GPU_LAYERS` | `0` (or higher on GPU hardware) |
+
+| Variable            | Value                             |
+| ------------------- | --------------------------------- |
+| `INFERENCE_BACKEND` | `llama_cpp`                       |
+| `MODEL_REPO`        | `Qwen/Qwen2.5-3B-Instruct-GGUF`   |
+| `MODEL_FILE`        | `qwen2.5-3b-instruct-q4_k_m.gguf` |
+| `N_CTX`             | `4096`                            |
+| `N_GPU_LAYERS`      | `0` (or higher on GPU hardware)   |
+
 
 ### 5. Build and verify
 
@@ -177,14 +183,16 @@ If cold starts are too slow, attach a **Storage Bucket** in Space settings so do
 
 ## Troubleshooting
 
-| Symptom | Likely cause | Fix |
-|---------|--------------|-----|
-| First chat hangs / slow | GGUF downloading from Hub | Pre-download locally; on Space, wait or use Storage Bucket |
-| `Failed to load model` in chat | Wrong `MODEL_REPO` / `MODEL_FILE` | Check env vars match a valid GGUF on Hub |
-| Docker build fails on `llama-cpp-python` | Missing build tools | Dockerfile already installs `build-essential` and `cmake` |
-| Space build fails | Missing `uv.lock` or README YAML | Ensure `sdk: docker` is in root `README.md` frontmatter |
-| `transformers` backend error | Optional deps not installed | Run `uv sync --package inference --extra transformers` |
-| Port already in use locally | Another process on 7860 | `PORT=7861 uv run --package gradio-space python -m gradio_space.app` |
+
+| Symptom                                  | Likely cause                      | Fix                                                                  |
+| ---------------------------------------- | --------------------------------- | -------------------------------------------------------------------- |
+| First chat hangs / slow                  | GGUF downloading from Hub         | Pre-download locally; on Space, wait or use Storage Bucket           |
+| `Failed to load model` in chat           | Wrong `MODEL_REPO` / `MODEL_FILE` | Check env vars match a valid GGUF on Hub                             |
+| Docker build fails on `llama-cpp-python` | Missing build tools               | Dockerfile already installs `build-essential` and `cmake`            |
+| Space build fails                        | Missing `uv.lock` or README YAML  | Ensure `sdk: docker` is in root `README.md` frontmatter              |
+| `transformers` backend error             | Optional deps not installed       | Run `uv sync --package inference --extra transformers`               |
+| Port already in use locally              | Another process on 7860           | `PORT=7861 uv run --package gradio-space python -m gradio_space.app` |
+
 
 ---
 
@@ -196,8 +204,11 @@ All three environments use the same command:
 uv run --package gradio-space python -m gradio_space.app
 ```
 
-| Environment | How to run |
-|-------------|------------|
-| Local dev | `uv run --package gradio-space python -m gradio_space.app` |
-| Docker | `docker run -p 7860:7860 hackathon-space` |
-| HF Space | Built and started automatically from `Dockerfile` `CMD` |
+
+| Environment | How to run                                                 |
+| ----------- | ---------------------------------------------------------- |
+| Local dev   | `uv run --package gradio-space python -m gradio_space.app` |
+| Docker      | `docker run -p 7860:7860 hackathon-space`                  |
+| HF Space    | Built and started automatically from `Dockerfile` `CMD`    |
+
+

apps/gradio-space/src/gradio_space/app.py

@@ -2,36 +2,39 @@ import os
 
 import gradio as gr
 
-from inference.factory import get_backend
+from inference.config import get_app_config, get_model_config
+from inference.factory import get_backend, reset_backend
 
-_backend = get_backend()
-_model_ready = False
-_load_error: str | None = None
+_app_config = get_app_config()
+_current_model_key: str | None = None
+_load_state: dict[str, bool] = {}
+_load_errors: dict[str, str] = {}
 
 
-def _ensure_model_loaded() -> str | None:
-    global _model_ready, _load_error
+def _ensure_model_loaded(model_key: str) -> str | None:
+    global _current_model_key
 
-    if _model_ready:
+    if model_key != _current_model_key:
+        reset_backend()
+        _current_model_key = model_key
+
+    if _load_state.get(model_key):
         return None
 
-    if _load_error:
-        return _load_error
+    if model_key in _load_errors:
+        return _load_errors[model_key]
 
     try:
-        _backend.load()
-        _model_ready = True
+        get_backend(model_key).load()
+        _load_state[model_key] = True
         return None
     except Exception as exc:  # noqa: BLE001 — surface model load failures in the UI
-        _load_error = f"Failed to load model: {exc}"
-        return _load_error
-
+        message = f"Failed to load model: {exc}"
+        _load_errors[model_key] = message
+        return message
 
-def chat(message: str, history: list) -> str:
-    load_error = _ensure_model_loaded()
-    if load_error:
-        return load_error
 
+def _history_to_messages(history: list) -> list[dict[str, str]]:
     messages: list[dict[str, str]] = []
     for item in history:
         if isinstance(item, dict):
@@ -41,48 +44,116 @@ def chat(message: str, history: list) -> str:
             messages.append({"role": "user", "content": user_msg})
             if assistant_msg:
                 messages.append({"role": "assistant", "content": assistant_msg})
+    return messages
 
-    messages.append({"role": "user", "content": message})
-    return _backend.chat(messages)
 
+def chat(message: str, history: list, model_key: str) -> str:
+    load_error = _ensure_model_loaded(model_key)
+    if load_error:
+        return load_error
 
-def warmup() -> str:
-    if _model_ready:
-        return "Model ready."
+    messages = _history_to_messages(history)
+    messages.append({"role": "user", "content": message})
+    return get_backend(model_key).chat(messages)
+
+
+def _runtime_device_hint(model_key: str) -> str:
+    model = get_model_config(model_key)
+    if model.backend == "transformers":
+        try:
+            import torch
+
+            if torch.cuda.is_available():
+                return f"GPU ({torch.cuda.get_device_name(0)})"
+        except ImportError:
+            pass
+        return "CPU"
+    if model.n_gpu_layers > 0:
+        return f"llama.cpp GPU offload ({model.n_gpu_layers} layers)"
+    return "CPU"
+
+
+def warmup(model_key: str | None = None) -> str:
+    key = model_key or _app_config.active_model
+    model = get_model_config(key)
+
+    if _load_state.get(key):
+        backend = get_backend(key)
+        device = (
+            backend.device_label
+            if hasattr(backend, "device_label")
+            else _runtime_device_hint(key)
+        )
+        return f"Model ready: {model.label} on {device}"
 
-    if _load_error:
-        return _load_error
+    if key in _load_errors:
+        return _load_errors[key]
 
+    device_hint = _runtime_device_hint(key)
     return (
-        "Model not loaded yet. It will download from Hugging Face Hub on the "
-        "first chat message — this can take a few minutes on CPU."
+        f"Preset `{key}` selected ({model.backend}, {device_hint}). "
+        "Weights load on the first chat message."
     )
 
 
+def model_status(model_key: str) -> str:
+    model = get_model_config(model_key)
+    return f"**{model.label}**\n\n- Backend: `{model.backend}`\n- {warmup(model_key)}"
+
+
 def build_demo() -> gr.Blocks:
-    model_repo = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF")
-    model_file = os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf")
-    backend_name = os.environ.get("INFERENCE_BACKEND", "llama_cpp")
+    active = _app_config.active
+    presets_note = (
+        f"Presets file: `{_app_config.presets_path}`"
+        if _app_config.presets_path
+        else "Using built-in presets (models.yaml not found)."
+    )
 
     with gr.Blocks(title="Small Model Hackathon") as demo:
         gr.Markdown(
             f"""
 # Small Model Chat
 
-Local inference via **{backend_name}**.
+Local inference with preset-based configuration.
 
-- **Repo:** `{model_repo}`
-- **File:** `{model_file}`
+- **Default preset:** `{active.key}` — {active.label}
+- **Backend:** `{active.backend}`
+- {presets_note}
 
 Part of the [Build Small Hackathon](https://huggingface.co/build-small-hackathon).
 """
         )
-        status = gr.Markdown(warmup())
-        gr.ChatInterface(
-            fn=chat,
-            examples=["Hello! What can you help me with?", "Explain llama.cpp in one sentence."],
-        )
-        demo.load(warmup, outputs=status)
+
+        if _app_config.allow_model_switch and len(_app_config.models) > 1:
+            model_dropdown = gr.Dropdown(
+                choices=_app_config.model_choices(),
+                value=_app_config.active_model,
+                label="Model preset",
+                info="Switch presets for local testing. Each preset loads on first use.",
+            )
+            status = gr.Markdown(model_status(_app_config.active_model))
+
+            model_dropdown.change(
+                fn=model_status,
+                inputs=model_dropdown,
+                outputs=status,
+            )
+
+            gr.ChatInterface(
+                fn=chat,
+                additional_inputs=[model_dropdown],
+                examples=[
+                    ["Hello! What can you help me with?", _app_config.active_model],
+                    ["Explain llama.cpp in one sentence.", _app_config.active_model],
+                ],
+            )
+        else:
+            status = gr.Markdown(model_status(_app_config.active_model))
+            gr.ChatInterface(
+                fn=lambda message, history: chat(message, history, _app_config.active_model),
+                examples=["Hello! What can you help me with?", "Explain llama.cpp in one sentence."],
+            )
+            demo.load(lambda: warmup(_app_config.active_model), outputs=status)
 
     return demo
 

libs/inference/pyproject.toml

@@ -8,15 +8,13 @@ authors = [
 ]
 requires-python = ">=3.12"
 dependencies = [
+    "accelerate>=1.2.0",
     "huggingface-hub>=0.27.0",
     "llama-cpp-python>=0.3.0",
-]
-
-[project.optional-dependencies]
-transformers = [
-    "accelerate>=1.2.0",
+    "pyyaml>=6.0.2",
     "torch>=2.5.0",
-    "transformers>=4.47.0",
+    "torchvision>=0.20.0",
+    "transformers>=5.7.0",
 ]
 
 [build-system]

libs/inference/src/inference/__init__.py

@@ -1,3 +1,12 @@
-from inference.factory import get_backend
+from inference.config import AppConfig, ModelConfig, get_app_config, get_model_config, load_app_config
+from inference.factory import get_backend, reset_backend
 
-__all__ = ["get_backend"]
+__all__ = [
+    "AppConfig",
+    "ModelConfig",
+    "get_app_config",
+    "get_backend",
+    "get_model_config",
+    "load_app_config",
+    "reset_backend",
+]

libs/inference/src/inference/config.py

@@ -0,0 +1,284 @@
+"""Central model preset and app configuration."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any, Literal
+
+BackendName = Literal["llama_cpp", "transformers"]
+
+DEFAULT_PRESET_KEY = "qwen3b-gguf"
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    """Single model preset used by inference backends and the Gradio UI."""
+
+    key: str
+    label: str
+    backend: BackendName
+    model_repo: str | None = None
+    model_file: str | None = None
+    model_path: str | None = None
+    model_id: str | None = None
+    trust_remote_code: bool = False
+    multimodal: bool = False
+    n_ctx: int = 4096
+    n_gpu_layers: int = 0
+    max_tokens: int = 512
+    temperature: float = 0.7
+
+    def cache_key(self) -> tuple[Any, ...]:
+        return (
+            self.backend,
+            self.model_repo,
+            self.model_file,
+            self.model_path,
+            self.model_id,
+            self.trust_remote_code,
+            self.multimodal,
+            self.n_ctx,
+            self.n_gpu_layers,
+        )
+
+    def summary(self) -> str:
+        if self.backend == "llama_cpp":
+            source = self.model_path or f"{self.model_repo}/{self.model_file}"
+            return f"{self.label} · llama.cpp · {source}"
+        return f"{self.label} · transformers · {self.model_id}"
+
+    def resolve_paths(self, base_dir: Path) -> ModelConfig:
+        updates: dict[str, Any] = {}
+
+        if self.model_path:
+            path = Path(self.model_path)
+            if not path.is_absolute():
+                updates["model_path"] = str((base_dir / path).resolve())
+
+        if self.model_id and self.model_id.startswith(("./", "../")):
+            updates["model_id"] = str((base_dir / self.model_id).resolve())
+
+        return replace(self, **updates) if updates else self
+
+
+@dataclass(frozen=True)
+class AppConfig:
+    """Runtime app configuration for dev and Hugging Face Space."""
+
+    active_model: str
+    models: dict[str, ModelConfig]
+    allow_model_switch: bool = False
+    model_cache_dir: str | None = None
+    presets_path: Path | None = None
+
+    def get_model(self, key: str | None = None) -> ModelConfig:
+        model_key = key or self.active_model
+        if model_key not in self.models:
+            known = ", ".join(sorted(self.models))
+            raise KeyError(f"Unknown model preset {model_key!r}. Known presets: {known}")
+        return self.models[model_key]
+
+    @property
+    def active(self) -> ModelConfig:
+        return self.get_model(self.active_model)
+
+    def model_choices(self) -> list[tuple[str, str]]:
+        return [(model.label, model.key) for model in self.models.values()]
+
+
+def _builtin_presets() -> dict[str, ModelConfig]:
+    return {
+        DEFAULT_PRESET_KEY: ModelConfig(
+            key=DEFAULT_PRESET_KEY,
+            label="Qwen 2.5 3B Instruct (GGUF)",
+            backend="llama_cpp",
+            model_repo="Qwen/Qwen2.5-3B-Instruct-GGUF",
+            model_file="qwen2.5-3b-instruct-q4_k_m.gguf",
+        ),
+        "minicpm5-1b": ModelConfig(
+            key="minicpm5-1b",
+            label="MiniCPM5 1B (Transformers)",
+            backend="transformers",
+            model_id="openbmb/MiniCPM5-1B",
+            trust_remote_code=True,
+        ),
+        "gemma-merged-local": ModelConfig(
+            key="gemma-merged-local",
+            label="Fine-tuned merged model (local)",
+            backend="transformers",
+            model_id="./gemma_merged_model",
+        ),
+    }
+
+
+def _find_presets_path() -> Path | None:
+    env_path = os.environ.get("MODEL_PRESETS_PATH")
+    if env_path:
+        path = Path(env_path)
+        if path.is_file():
+            return path.resolve()
+
+    for base in (Path.cwd(), *Path.cwd().parents):
+        candidate = base / "models.yaml"
+        if candidate.is_file():
+            return candidate.resolve()
+    return None
+
+
+def _repo_root_for(presets_path: Path | None) -> Path:
+    if presets_path is not None:
+        return presets_path.parent
+    app_root = os.environ.get("APP_ROOT")
+    if app_root:
+        return Path(app_root).resolve()
+    return Path.cwd().resolve()
+
+
+def _parse_model_entry(key: str, raw: dict[str, Any]) -> ModelConfig:
+    backend = str(raw.get("backend", "llama_cpp")).lower()
+    if backend not in ("llama_cpp", "transformers"):
+        raise ValueError(f"Preset {key!r}: backend must be llama_cpp or transformers")
+
+    return ModelConfig(
+        key=key,
+        label=str(raw.get("label", key)),
+        backend=backend,  # type: ignore[arg-type]
+        model_repo=raw.get("model_repo"),
+        model_file=raw.get("model_file"),
+        model_path=raw.get("model_path"),
+        model_id=raw.get("model_id"),
+        trust_remote_code=bool(raw.get("trust_remote_code", False)),
+        multimodal=bool(raw.get("multimodal", False)),
+        n_ctx=int(raw.get("n_ctx", 4096)),
+        n_gpu_layers=int(raw.get("n_gpu_layers", 0)),
+        max_tokens=int(raw.get("max_tokens", 512)),
+        temperature=float(raw.get("temperature", 0.7)),
+    )
+
+
+def _load_presets_from_yaml(path: Path) -> tuple[dict[str, Any], dict[str, ModelConfig]]:
+    try:
+        import yaml
+    except ImportError as exc:
+        raise ImportError(
+            "Loading models.yaml requires PyYAML. Install with: uv add --package inference pyyaml"
+        ) from exc
+
+    data = yaml.safe_load(path.read_text()) or {}
+    defaults = data.get("defaults", {})
+    raw_models = data.get("models", {})
+    if not isinstance(raw_models, dict) or not raw_models:
+        raise ValueError(f"{path}: expected non-empty top-level 'models' mapping")
+
+    models = {key: _parse_model_entry(key, value) for key, value in raw_models.items()}
+    return defaults, models
+
+
+def _apply_legacy_env_overrides(model: ModelConfig) -> ModelConfig:
+    """Keep single-model .env workflow working alongside preset keys."""
+
+    updates: dict[str, Any] = {}
+
+    backend = os.environ.get("INFERENCE_BACKEND")
+    if backend:
+        updates["backend"] = backend.lower()
+
+    for field, env_name in (
+        ("model_repo", "MODEL_REPO"),
+        ("model_file", "MODEL_FILE"),
+        ("model_path", "MODEL_PATH"),
+        ("model_id", "MODEL_ID"),
+    ):
+        value = os.environ.get(env_name)
+        if value:
+            updates[field] = value
+
+    if os.environ.get("TRUST_REMOTE_CODE", "").lower() in {"1", "true", "yes"}:
+        updates["trust_remote_code"] = True
+
+    for field, env_name in (
+        ("n_ctx", "N_CTX"),
+        ("n_gpu_layers", "N_GPU_LAYERS"),
+        ("max_tokens", "MAX_TOKENS"),
+    ):
+        value = os.environ.get(env_name)
+        if value is not None and value != "":
+            updates[field] = int(value)
+
+    temperature = os.environ.get("TEMPERATURE")
+    if temperature is not None and temperature != "":
+        updates["temperature"] = float(temperature)
+
+    return replace(model, **updates) if updates else model
+
+
+def load_app_config() -> AppConfig:
+    presets_path = _find_presets_path()
+    repo_root = _repo_root_for(presets_path)
+
+    if presets_path is None:
+        defaults: dict[str, Any] = {}
+        models = _builtin_presets()
+    else:
+        defaults, models = _load_presets_from_yaml(presets_path)
+
+    active_model = os.environ.get("ACTIVE_MODEL") or defaults.get(
+        "active_model", DEFAULT_PRESET_KEY
+    )
+    if active_model not in models:
+        active_model = next(iter(models))
+
+    allow_model_switch = os.environ.get("ALLOW_MODEL_SWITCH")
+    if allow_model_switch is None:
+        allow_switch = bool(defaults.get("allow_model_switch", False))
+    else:
+        allow_switch = allow_model_switch.lower() in {"1", "true", "yes"}
+
+    cache_dir = os.environ.get("MODEL_CACHE_DIR") or defaults.get("model_cache_dir")
+
+    resolved_models = {
+        key: model.resolve_paths(repo_root) for key, model in models.items()
+    }
+
+    has_legacy_override = any(
+        os.environ.get(name)
+        for name in (
+            "INFERENCE_BACKEND",
+            "MODEL_REPO",
+            "MODEL_FILE",
+            "MODEL_PATH",
+            "MODEL_ID",
+            "N_CTX",
+            "N_GPU_LAYERS",
+        )
+    )
+    if has_legacy_override:
+        resolved_models[active_model] = _apply_legacy_env_overrides(
+            resolved_models[active_model]
+        )
+
+    return AppConfig(
+        active_model=active_model,
+        models=resolved_models,
+        allow_model_switch=allow_switch,
+        model_cache_dir=cache_dir,
+        presets_path=presets_path,
+    )
+
+
+_app_config: AppConfig | None = None
+
+
+def get_app_config(reload: bool = False) -> AppConfig:
+    global _app_config
+    if _app_config is None or reload:
+        _app_config = load_app_config()
+    return _app_config
+
+
+def get_model_config(key: str | None = None) -> ModelConfig:
+    config = get_app_config()
+    model_key = key or config.active_model
+    return config.get_model(model_key)

libs/inference/src/inference/factory.py

@@ -1,23 +1,40 @@
-import os
-from functools import lru_cache
-
 from inference.base import InferenceBackend
+from inference.config import ModelConfig, get_model_config
 from inference.llama_cpp import LlamaCppBackend
 
+_backend: InferenceBackend | None = None
+_backend_key: tuple | None = None
 
-@lru_cache(maxsize=1)
-def get_backend() -> InferenceBackend:
-    backend_name = os.environ.get("INFERENCE_BACKEND", "llama_cpp").lower()
 
-    if backend_name == "llama_cpp":
-        return LlamaCppBackend()
+def _create_backend(config: ModelConfig) -> InferenceBackend:
+    if config.backend == "llama_cpp":
+        return LlamaCppBackend(config)
 
-    if backend_name == "transformers":
+    if config.backend == "transformers":
         from inference.transformers import TransformersBackend
 
-        return TransformersBackend()
+        return TransformersBackend(config)
 
     raise ValueError(
-        f"Unknown INFERENCE_BACKEND={backend_name!r}. "
+        f"Unknown backend {config.backend!r} for preset {config.key!r}. "
         "Expected 'llama_cpp' or 'transformers'."
     )
+
+
+def get_backend(model_key: str | None = None) -> InferenceBackend:
+    global _backend, _backend_key
+
+    config = get_model_config(model_key)
+    cache_key = config.cache_key()
+
+    if _backend is None or _backend_key != cache_key:
+        _backend = _create_backend(config)
+        _backend_key = cache_key
+
+    return _backend
+
+
+def reset_backend() -> None:
+    global _backend, _backend_key
+    _backend = None
+    _backend_key = None

libs/inference/src/inference/llama_cpp.py

@@ -4,31 +4,32 @@ from pathlib import Path
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
 
-
-DEFAULT_MODEL_REPO = "Qwen/Qwen2.5-3B-Instruct-GGUF"
-DEFAULT_MODEL_FILE = "qwen2.5-3b-instruct-q4_k_m.gguf"
+from inference.config import ModelConfig
 
 
 class LlamaCppBackend:
-    def __init__(self) -> None:
+    def __init__(self, config: ModelConfig) -> None:
+        self._config = config
         self._model: Llama | None = None
         self._model_path: str | None = None
 
     def _resolve_model_path(self) -> str:
-        model_path = os.environ.get("MODEL_PATH")
-        if model_path:
-            path = Path(model_path)
+        if self._config.model_path:
+            path = Path(self._config.model_path)
             if not path.exists():
-                raise FileNotFoundError(f"MODEL_PATH does not exist: {model_path}")
+                raise FileNotFoundError(f"MODEL_PATH does not exist: {self._config.model_path}")
             return str(path)
 
-        model_repo = os.environ.get("MODEL_REPO", DEFAULT_MODEL_REPO)
-        model_file = os.environ.get("MODEL_FILE", DEFAULT_MODEL_FILE)
+        if not self._config.model_repo or not self._config.model_file:
+            raise ValueError(
+                f"Preset {self._config.key!r} requires model_repo and model_file for llama_cpp"
+            )
+
         cache_dir = os.environ.get("MODEL_CACHE_DIR")
 
         return hf_hub_download(
-            repo_id=model_repo,
-            filename=model_file,
+            repo_id=self._config.model_repo,
+            filename=self._config.model_file,
             cache_dir=cache_dir,
         )
 
@@ -37,13 +38,11 @@ class LlamaCppBackend:
             return
 
         self._model_path = self._resolve_model_path()
-        n_ctx = int(os.environ.get("N_CTX", "4096"))
-        n_gpu_layers = int(os.environ.get("N_GPU_LAYERS", "0"))
 
         self._model = Llama(
             model_path=self._model_path,
-            n_ctx=n_ctx,
-            n_gpu_layers=n_gpu_layers,
+            n_ctx=self._config.n_ctx,
+            n_gpu_layers=self._config.n_gpu_layers,
             verbose=False,
         )
 
@@ -51,16 +50,16 @@ class LlamaCppBackend:
         self,
         prompt: str,
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
 
         result = self._model(
             prompt,
-            max_tokens=max_tokens,
-            temperature=temperature,
+            max_tokens=max_tokens or self._config.max_tokens,
+            temperature=temperature if temperature is not None else self._config.temperature,
             echo=False,
         )
         return result["choices"][0]["text"].strip()
@@ -69,15 +68,15 @@ class LlamaCppBackend:
         self,
         messages: list[dict[str, str]],
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
 
         result = self._model.create_chat_completion(
             messages=messages,
-            max_tokens=max_tokens,
-            temperature=temperature,
+            max_tokens=max_tokens or self._config.max_tokens,
+            temperature=temperature if temperature is not None else self._config.temperature,
         )
         return result["choices"][0]["message"]["content"].strip()

libs/inference/src/inference/transformers.py

@@ -1,72 +1,136 @@
-import os
+from __future__ import annotations
 
-from inference.base import InferenceBackend
+from inference.config import ModelConfig
 
 
 class TransformersBackend:
-    def __init__(self) -> None:
+    def __init__(self, config: ModelConfig) -> None:
+        self._config = config
         self._model = None
         self._tokenizer = None
+        self._processor = None
+        self._device_label: str | None = None
+
+    def _resolve_device(self):
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda", torch.float16, "auto"
+        return "cpu", torch.float32, None
 
     def load(self) -> None:
         if self._model is not None:
             return
 
+        if not self._config.model_id:
+            raise ValueError(
+                f"Preset {self._config.key!r} requires model_id for transformers backend"
+            )
+
         try:
             import torch
-            from transformers import AutoModelForCausalLM, AutoTokenizer
+            from transformers import (
+                AutoModelForCausalLM,
+                AutoModelForImageTextToText,
+                AutoProcessor,
+                AutoTokenizer,
+            )
         except ImportError as exc:
             raise ImportError(
-                "transformers backend requires optional deps. "
-                "Install with: uv sync --package inference --extra transformers"
+                "transformers backend requires torch and transformers. "
+                "Install with: uv sync --all-packages"
             ) from exc
 
-        model_id = os.environ.get("MODEL_ID", "Qwen/Qwen2.5-3B-Instruct")
-        device = "cuda" if torch.cuda.is_available() else "cpu"
-
-        self._tokenizer = AutoTokenizer.from_pretrained(model_id)
-        self._model = AutoModelForCausalLM.from_pretrained(
-            model_id,
-            torch_dtype=torch.float16 if device == "cuda" else torch.float32,
-            device_map="auto" if device == "cuda" else None,
+        device, torch_dtype, device_map = self._resolve_device()
+        self._device_label = (
+            f"cuda ({torch.cuda.get_device_name(0)})"
+            if device == "cuda"
+            else "cpu"
         )
+
+        common_kwargs = {
+            "trust_remote_code": self._config.trust_remote_code,
+        }
+        model_kwargs = {
+            **common_kwargs,
+            "dtype": torch_dtype,
+            "device_map": device_map,
+        }
+
+        if self._config.multimodal:
+            self._processor = AutoProcessor.from_pretrained(
+                self._config.model_id,
+                **common_kwargs,
+            )
+            self._model = AutoModelForImageTextToText.from_pretrained(
+                self._config.model_id,
+                **model_kwargs,
+            )
+        else:
+            self._tokenizer = AutoTokenizer.from_pretrained(
+                self._config.model_id,
+                **common_kwargs,
+            )
+            self._model = AutoModelForCausalLM.from_pretrained(
+                self._config.model_id,
+                **model_kwargs,
+            )
+
         if device == "cpu":
             self._model.to(device)
 
-    def generate(
-        self,
-        prompt: str,
-        *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
-    ) -> str:
+    @property
+    def device_label(self) -> str:
         self.load()
-        assert self._model is not None
-        assert self._tokenizer is not None
+        return self._device_label or "unknown"
 
-        import torch
+    def _normalize_messages(
+        self, messages: list[dict[str, str]]
+    ) -> list[dict[str, object]]:
+        if not self._config.multimodal:
+            return messages
 
-        inputs = self._tokenizer(prompt, return_tensors="pt").to(self._model.device)
-        output = self._model.generate(
-            **inputs,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            do_sample=temperature > 0,
-        )
-        generated = output[0][inputs["input_ids"].shape[-1] :]
-        return self._tokenizer.decode(generated, skip_special_tokens=True).strip()
+        normalized: list[dict[str, object]] = []
+        for message in messages:
+            content = message["content"]
+            if isinstance(content, str):
+                content = [{"type": "text", "text": content}]
+            normalized.append({"role": message["role"], "content": content})
+        return normalized
 
-    def chat(
+    def _generate_from_messages(
         self,
-        messages: list[dict[str, str]],
+        messages: list[dict[str, object]],
         *,
-        max_tokens: int = 512,
-        temperature: float = 0.7,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
     ) -> str:
         self.load()
         assert self._model is not None
-        assert self._tokenizer is not None
 
+        max_new_tokens = max_tokens or self._config.max_tokens
+        temp = self._config.temperature if temperature is None else temperature
+
+        if self._config.multimodal:
+            assert self._processor is not None
+            inputs = self._processor.apply_chat_template(
+                messages,
+                tokenize=True,
+                add_generation_prompt=True,
+                return_dict=True,
+                return_tensors="pt",
+            )
+            inputs = inputs.to(self._model.device)
+            output = self._model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                temperature=temp,
+                do_sample=temp > 0,
+            )
+            generated = output[0][inputs["input_ids"].shape[-1] :]
+            return self._processor.decode(generated, skip_special_tokens=True).strip()
+
+        assert self._tokenizer is not None
         if hasattr(self._tokenizer, "apply_chat_template"):
             prompt = self._tokenizer.apply_chat_template(
                 messages,
@@ -76,14 +140,45 @@ class TransformersBackend:
         else:
             parts = []
             for message in messages:
-                role = message["role"]
-                content = message["content"]
+                role = str(message["role"])
+                content = str(message["content"])
                 parts.append(f"{role}: {content}")
             parts.append("assistant:")
             prompt = "\n".join(parts)
 
-        return self.generate(prompt, max_tokens=max_tokens, temperature=temperature)
+        inputs = self._tokenizer(prompt, return_tensors="pt").to(self._model.device)
+        output = self._model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            temperature=temp,
+            do_sample=temp > 0,
+        )
+        generated = output[0][inputs["input_ids"].shape[-1] :]
+        return self._tokenizer.decode(generated, skip_special_tokens=True).strip()
 
+    def generate(
+        self,
+        prompt: str,
+        *,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> str:
+        return self.chat(
+            [{"role": "user", "content": prompt}],
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )
 
-# Satisfy static type checkers that expect InferenceBackend.
-_: InferenceBackend = TransformersBackend()
+    def chat(
+        self,
+        messages: list[dict[str, str]],
+        *,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> str:
+        normalized = self._normalize_messages(messages)
+        return self._generate_from_messages(
+            normalized,
+            max_tokens=max_tokens,
+            temperature=temperature,
+        )

libs/inference/tests/test_config.py

@@ -0,0 +1,76 @@
+from pathlib import Path
+
+import pytest
+
+from inference.config import load_app_config
+
+
+def test_load_app_config_from_models_yaml(tmp_path, monkeypatch):
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        """
+defaults:
+  active_model: demo
+  allow_model_switch: true
+models:
+  demo:
+    label: Demo preset
+    backend: llama_cpp
+    model_repo: org/model-GGUF
+    model_file: demo.gguf
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.delenv("ACTIVE_MODEL", raising=False)
+
+    config = load_app_config()
+
+    assert config.active_model == "demo"
+    assert config.allow_model_switch is True
+    assert config.get_model("demo").model_repo == "org/model-GGUF"
+
+
+def test_legacy_env_overrides_active_preset(tmp_path, monkeypatch):
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        """
+defaults:
+  active_model: demo
+models:
+  demo:
+    label: Demo
+    backend: llama_cpp
+    model_repo: org/original
+    model_file: original.gguf
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("MODEL_REPO", "org/override")
+    monkeypatch.setenv("MODEL_FILE", "override.gguf")
+
+    model = load_app_config().get_model("demo")
+
+    assert model.model_repo == "org/override"
+    assert model.model_file == "override.gguf"
+
+
+def test_resolve_relative_model_path(tmp_path, monkeypatch):
+    local_dir = tmp_path / "gemma_merged_model"
+    local_dir.mkdir()
+    presets = tmp_path / "models.yaml"
+    presets.write_text(
+        f"""
+defaults:
+  active_model: local
+models:
+  local:
+    label: Local merged
+    backend: transformers
+    model_id: ./{local_dir.name}
+"""
+    )
+    monkeypatch.chdir(tmp_path)
+
+    model = load_app_config().get_model("local")
+
+    assert model.model_id == str(local_dir.resolve())

models.yaml

@@ -0,0 +1,54 @@
+# Model preset registry for dev and Hugging Face Space.
+# Select active preset with ACTIVE_MODEL; override any field via .env (see .env.example).
+
+defaults:
+  active_model: minicpm-v-4.6
+  # Dev: set ALLOW_MODEL_SWITCH=true in .env to expose a dropdown in Gradio.
+  # Space: keep false so visitors use one pinned model.
+  allow_model_switch: false
+
+models:
+  minicpm-v-4.6:
+    label: MiniCPM-V 4.6 (Transformers, ~0.8B, default)
+    backend: transformers
+    model_id: openbmb/MiniCPM-V-4.6
+    trust_remote_code: true
+    multimodal: true
+
+  qwen3b-gguf:
+    label: Qwen 2.5 3B Instruct (GGUF)
+    backend: llama_cpp
+    model_repo: Qwen/Qwen2.5-3B-Instruct-GGUF
+    model_file: qwen2.5-3b-instruct-q4_k_m.gguf
+    n_ctx: 4096
+    n_gpu_layers: 0
+
+  llama32-3b-gguf:
+    label: Llama 3.2 3B Instruct (GGUF)
+    backend: llama_cpp
+    model_repo: bartowski/Llama-3.2-3B-Instruct-GGUF
+    model_file: Llama-3.2-3B-Instruct-Q4_K_M.gguf
+    n_ctx: 4096
+    n_gpu_layers: 0
+
+  minicpm5-1b:
+    label: MiniCPM5 1B (Transformers)
+    backend: transformers
+    model_id: openbmb/MiniCPM5-1B
+    trust_remote_code: true
+
+  gemma4-e2b-mobile:
+    label: Gemma 4 E2B IT QAT Mobile (Transformers)
+    backend: transformers
+    model_id: google/gemma-4-E2B-it-qat-mobile-transformers
+    trust_remote_code: true
+
+  gemma-merged-local:
+    label: Fine-tuned merged model (local path)
+    backend: transformers
+    model_id: ./gemma_merged_model
+
+  gemma-lora-local:
+    label: Fine-tuned LoRA adapter (local path)
+    backend: transformers
+    model_id: ./gemma_finetuned_model

notebook/gemma-finetune.ipynb

@@ -0,0 +1,499 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab_type": "code",
+        "execution": {
+          "iopub.execute_input": "2026-06-07T13:53:33.681800Z",
+          "iopub.status.busy": "2026-06-07T13:53:33.681410Z",
+          "iopub.status.idle": "2026-06-07T13:53:47.949186Z",
+          "shell.execute_reply": "2026-06-07T13:53:47.948138Z",
+          "shell.execute_reply.started": "2026-06-07T13:53:33.681762Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "!pip install -U transformers"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text"
+      },
+      "source": [
+        "## Local Inference on GPU \n",
+        "Model page: https://huggingface.co/google/gemma-4-E2B-it-qat-mobile-transformers\n",
+        "\n",
+        "⚠️ If the generated code snippets do not work, please open an issue on either the [model repo](https://huggingface.co/google/gemma-4-E2B-it-qat-mobile-transformers)\n",
+        "\t\t\tand/or on [huggingface.js](https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries-snippets.ts) 🙏"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T14:00:05.743785Z",
+          "iopub.status.busy": "2026-06-07T14:00:05.742974Z",
+          "iopub.status.idle": "2026-06-07T14:00:05.748408Z",
+          "shell.execute_reply": "2026-06-07T14:00:05.747778Z",
+          "shell.execute_reply.started": "2026-06-07T14:00:05.743756Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "# Load model directly\n",
+        "from transformers import AutoModel\n",
+        "import torch\n",
+        "\n",
+        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+        "\n",
+        "model_name_gemma=\"google/gemma-4-E2B-it-qat-mobile-transformers\"\n",
+        "model_name=\"openbmb/MiniCPM5-1B\"\n",
+        "model_openbmb=\"openbmb/MiniCPM5-1B\""
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab_type": "code",
+        "execution": {
+          "iopub.execute_input": "2026-06-07T13:58:21.771701Z",
+          "iopub.status.busy": "2026-06-07T13:58:21.770950Z",
+          "iopub.status.idle": "2026-06-07T13:58:42.868203Z",
+          "shell.execute_reply": "2026-06-07T13:58:42.867564Z",
+          "shell.execute_reply.started": "2026-06-07T13:58:21.771669Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "\n",
+        "model = AutoModel.from_pretrained(model_name, dtype=\"auto\").to(device)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T14:00:17.480009Z",
+          "iopub.status.busy": "2026-06-07T14:00:17.479002Z",
+          "iopub.status.idle": "2026-06-07T14:00:28.838734Z",
+          "shell.execute_reply": "2026-06-07T14:00:28.837707Z",
+          "shell.execute_reply.started": "2026-06-07T14:00:17.479974Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "\n",
+        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
+        "\n",
+        "tokenizer = AutoTokenizer.from_pretrained(\"openbmb/MiniCPM5-1B\")\n",
+        "model = AutoModelForCausalLM.from_pretrained(\"openbmb/MiniCPM5-1B\")\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T14:00:51.808940Z",
+          "iopub.status.busy": "2026-06-07T14:00:51.808046Z",
+          "iopub.status.idle": "2026-06-07T14:00:59.481653Z",
+          "shell.execute_reply": "2026-06-07T14:00:59.480980Z",
+          "shell.execute_reply.started": "2026-06-07T14:00:51.808892Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "messages = [\n",
+        "    {\"role\": \"user\", \"content\": \"Who are you?\"},\n",
+        "]\n",
+        "inputs = tokenizer.apply_chat_template(\n",
+        "\tmessages,\n",
+        "\tadd_generation_prompt=True,\n",
+        "\ttokenize=True,\n",
+        "\treturn_dict=True,\n",
+        "\treturn_tensors=\"pt\",\n",
+        ").to(model.device)\n",
+        "\n",
+        "outputs = model.generate(**inputs, max_new_tokens=120)\n",
+        "print(tokenizer.decode(outputs[0][inputs[\"input_ids\"].shape[-1]:]))"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T13:59:02.169453Z",
+          "iopub.status.busy": "2026-06-07T13:59:02.168479Z",
+          "iopub.status.idle": "2026-06-07T13:59:04.394628Z",
+          "shell.execute_reply": "2026-06-07T13:59:04.393301Z",
+          "shell.execute_reply.started": "2026-06-07T13:59:02.169418Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "# from transformers import AutoTokenizer, pipeline\n",
+        "\n",
+        "# tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
+        "\n",
+        "# pipeline = pipeline(\n",
+        "#     \"text-generation\",\n",
+        "#     model=model,\n",
+        "#     tokenizer=tokenizer,\n",
+        "#     model_kwargs={\"torch_dtype\": \"auto\"},\n",
+        "# )\n",
+        "\n",
+        "# # Example of generating chat/text\n",
+        "# messages = [\n",
+        "#     {\"role\": \"user\", \"content\": \"What is your favorite color?\"},\n",
+        "# ]\n",
+        "\n",
+        "# prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
+        "# outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)\n",
+        "# print(outputs[0][\"generated_text\"])"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Finetuning"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T14:01:09.304174Z",
+          "iopub.status.busy": "2026-06-07T14:01:09.303286Z",
+          "iopub.status.idle": "2026-06-07T14:01:32.898893Z",
+          "shell.execute_reply": "2026-06-07T14:01:32.897942Z",
+          "shell.execute_reply.started": "2026-06-07T14:01:09.304140Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "# Install necessary libraries for finetuning (if not already installed)\n",
+        "!pip install -q -U accelerate peft bitsandbytes transformers trl datasets\n",
+        "\n",
+        "import torch\n",
+        "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
+        "from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
+        "from trl import SFTTrainer\n",
+        "from datasets import Dataset\n",
+        "\n",
+        "# --- 1. Load Model and Tokenizer for Finetuning with QLoRA ---\n",
+        "# The `model` variable from a previous cell is AutoModel, for finetuning\n",
+        "# we typically need AutoModelForCausalLM with BitsAndBytesConfig.\n",
+        "# Let's reload it for clarity in this finetuning example.\n",
+        "\n",
+        "# model_id = \"google/gemma-4-E2B-it-qat-mobile-transformers\"\n",
+        "model_id = model_openbmb\n",
+        "\n",
+        "# Configure 4-bit quantization\n",
+        "bnb_config = BitsAndBytesConfig(\n",
+        "    load_in_4bit=True,\n",
+        "    bnb_4bit_quant_type=\"nf4\",\n",
+        "    bnb_4bit_compute_dtype=torch.bfloat16,\n",
+        "    bnb_4bit_use_double_quant=False,\n",
+        ")\n",
+        "\n",
+        "# Load the model with 4-bit quantization\n",
+        "# Assuming 'model' from earlier cell is a suitable base for this if not reloaded\n",
+        "# If you've already loaded it as `AutoModel`, you might need to ensure it's `AutoModelForCausalLM`\n",
+        "# and can be quantizied. For a robust finetuning example, it's safer to load it here again.\n",
+        "\n",
+        "tuned_model = AutoModelForCausalLM.from_pretrained(\n",
+        "    model_id,\n",
+        "    quantization_config=bnb_config,\n",
+        "    device_map=\"auto\",\n",
+        ")\n",
+        "\n",
+        "tuned_model.config.use_cache = False\n",
+        "tuned_model.config.pretraining_tp = 1\n",
+        "\n",
+        "tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)\n",
+        "tokenizer.pad_token = tokenizer.eos_token\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "\n",
+        "# Prepare model for k-bit training\n",
+        "tuned_model = prepare_model_for_kbit_training(tuned_model)\n",
+        "\n",
+        "# --- 2. Configure LoRA ---\n",
+        "lora_config = LoraConfig(\n",
+        "    r=16, # LoRA attention dimension\n",
+        "    lora_alpha=16, # Alpha parameter for LoRA scaling\n",
+        "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"], # Target all linear layers\n",
+        "    lora_dropout=0.05, # Dropout probability for LoRA layers\n",
+        "    bias=\"none\", # Only add bias to the LoRA layers\n",
+        "    task_type=\"CAUSAL_LM\", # Task type for causal language modeling\n",
+        ")\n",
+        "\n",
+        "# Do NOT call get_peft_model() here — SFTTrainer wraps the model when peft_config is passed.\n",
+        "# tuned_model = get_peft_model(tuned_model, lora_config)\n",
+        "\n",
+        "# --- 3. Prepare a Sample Dataset ---\n",
+        "# For a real-world scenario, you would load your own dataset using `load_dataset`\n",
+        "# from the `datasets` library and format it appropriately.\n",
+        "# This is a simple dummy dataset for demonstration.\n",
+        "\n",
+        "# Example instruction tuning dataset format\n",
+        "data = {\n",
+        "    \"text\": [\n",
+        "        \"<start_of_turn>user\\nWhat is the capital of France?<end_of_turn>\\n<start_of_turn>model\\nParis is the capital of France.<end_of_turn>\",\n",
+        "        \"<start_of_turn>user\\nSuggest a healthy snack.\\n<end_of_turn>\\n<start_of_turn>model\\nAlmonds or a piece of fruit like an apple are great healthy snack options.<end_of_turn>\",\n",
+        "        \"<start_of_turn>user\\nExplain the concept of photosynthesis.\\n<end_of_turn>\\n<start_of_turn>model\\nPhotosynthesis is the process by which green plants and some other organisms convert light energy into chemical energy.<end_of_turn>\"\n",
+        "    ]\n",
+        "}\n",
+        "\n",
+        "dataset = Dataset.from_dict(data)\n",
+        "\n",
+        "# --- 4. Define Training Arguments ---\n",
+        "from transformers import TrainingArguments\n",
+        "\n",
+        "training_args = TrainingArguments(\n",
+        "    output_dir=\"./gemma_finetuned\", # Output directory for checkpoints and logs\n",
+        "    num_train_epochs=1, # Number of training epochs\n",
+        "    per_device_train_batch_size=2, # Batch size per GPU/CPU for training\n",
+        "    gradient_accumulation_steps=2, # Number of updates steps to accumulate before performing a backward/update pass\n",
+        "    optim=\"paged_adamw_8bit\", # Optimizer to use\n",
+        "    save_steps=100, # Save checkpoint every X updates steps\n",
+        "    logging_steps=10, # Log every X updates steps\n",
+        "    learning_rate=2e-4, # Initial learning rate for AdamW optimizer\n",
+        "    weight_decay=0.001, # Weight decay for AdamW\n",
+        "    fp16=False, # Must match bnb_4bit_compute_dtype (bf16 below)\n",
+        "    bf16=True,  # Use bf16 when bnb_4bit_compute_dtype=torch.bfloat16\n",
+        "    max_grad_norm=0.3, # Max gradient norm\n",
+        "    max_steps=-1, # Don't limit training by steps, use epochs\n",
+        "    warmup_ratio=0.03, # Ratio of total steps for a linear warmup from 0 to learning_rate\n",
+        "    # group_by_length=True, # Group sequences of roughly the same length together to speed up training\n",
+        "    lr_scheduler_type=\"constant\", # Learning rate scheduler type\n",
+        "    report_to=\"none\" # Disable reporting to any tracking service\n",
+        ")\n",
+        "\n",
+        "# --- 5. Initialize and Run SFTTrainer ---\n",
+        "\n",
+        "trainer = SFTTrainer(\n",
+        "    model=tuned_model,          # plain (non-PEFT) base model\n",
+        "    train_dataset=dataset,\n",
+        "    peft_config=lora_config,    # SFTTrainer applies LoRA internally\n",
+        "    # dataset_text_field=\"text\", # Name of the column containing the text data\n",
+        "    # tokenizer=tokenizer,\n",
+        "    args=training_args,\n",
+        "    # packing=False, # Whether to pack multiple short examples into one longer sequence to improve efficiency\n",
+        "    # max_seq_length=512, # Max sequence length to use for training\n",
+        ")\n",
+        "\n",
+        "print(\"Starting finetuning...\")\n",
+        "trainer.train()\n",
+        "print(\"Finetuning complete!\")\n",
+        "\n",
+        "# --- 6. Save the LoRA adapter ---\n",
+        "trainer.model.save_pretrained(\"./gemma_finetuned_model\")\n",
+        "tokenizer.save_pretrained(\"./gemma_finetuned_model\")\n",
+        "\n",
+        "# --- 7. (Optional) Merge LoRA adapters for inference ---\n",
+        "# Merge in-memory from the trained model (avoids AutoPeft reload + torchao version issues).\n",
+        "merged_model = trainer.model.merge_and_unload()\n",
+        "merged_model.save_pretrained(\"gemma_merged_model\", safe_serialization=True)\n",
+        "tokenizer.save_pretrained(\"gemma_merged_model\")\n",
+        "\n",
+        "# If you need to reload the adapter from disk later instead, upgrade torchao first:\n",
+        "# !pip install -U \"torchao>=0.16.0\"\n",
+        "# from peft import PeftModel\n",
+        "# base_model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map=\"auto\")\n",
+        "# peft_model = PeftModel.from_pretrained(base_model, \"./gemma_finetuned_model\")\n",
+        ""
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "execution": {
+          "iopub.execute_input": "2026-06-07T14:02:33.440153Z",
+          "iopub.status.busy": "2026-06-07T14:02:33.439372Z",
+          "iopub.status.idle": "2026-06-07T14:02:36.463743Z",
+          "shell.execute_reply": "2026-06-07T14:02:36.462765Z",
+          "shell.execute_reply.started": "2026-06-07T14:02:33.440118Z"
+        },
+        "trusted": true
+      },
+      "source": [
+        "\n",
+        "# Prepare model for k-bit training\n",
+        "tuned_model = prepare_model_for_kbit_training(tuned_model)\n",
+        "\n",
+        "# --- 2. Configure LoRA ---\n",
+        "lora_config = LoraConfig(\n",
+        "    r=16, # LoRA attention dimension\n",
+        "    lora_alpha=16, # Alpha parameter for LoRA scaling\n",
+        "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"], # Target all linear layers\n",
+        "    lora_dropout=0.05, # Dropout probability for LoRA layers\n",
+        "    bias=\"none\", # Only add bias to the LoRA layers\n",
+        "    task_type=\"CAUSAL_LM\", # Task type for causal language modeling\n",
+        ")\n",
+        "\n",
+        "# Do NOT call get_peft_model() here — SFTTrainer wraps the model when peft_config is passed.\n",
+        "# tuned_model = get_peft_model(tuned_model, lora_config)\n",
+        "\n",
+        "# --- 3. Prepare a Sample Dataset ---\n",
+        "# For a real-world scenario, you would load your own dataset using `load_dataset`\n",
+        "# from the `datasets` library and format it appropriately.\n",
+        "# This is a simple dummy dataset for demonstration.\n",
+        "\n",
+        "# Example instruction tuning dataset format\n",
+        "data = {\n",
+        "    \"text\": [\n",
+        "        \"<start_of_turn>user\\nWhat is the capital of France?<end_of_turn>\\n<start_of_turn>model\\nParis is the capital of France.<end_of_turn>\",\n",
+        "        \"<start_of_turn>user\\nSuggest a healthy snack.\\n<end_of_turn>\\n<start_of_turn>model\\nAlmonds or a piece of fruit like an apple are great healthy snack options.<end_of_turn>\",\n",
+        "        \"<start_of_turn>user\\nExplain the concept of photosynthesis.\\n<end_of_turn>\\n<start_of_turn>model\\nPhotosynthesis is the process by which green plants and some other organisms convert light energy into chemical energy.<end_of_turn>\"\n",
+        "    ]\n",
+        "}\n",
+        "\n",
+        "dataset = Dataset.from_dict(data)\n",
+        "\n",
+        "# --- 4. Define Training Arguments ---\n",
+        "from transformers import TrainingArguments\n",
+        "\n",
+        "training_args = TrainingArguments(\n",
+        "    output_dir=\"./gemma_finetuned\", # Output directory for checkpoints and logs\n",
+        "    num_train_epochs=1, # Number of training epochs\n",
+        "    per_device_train_batch_size=2, # Batch size per GPU/CPU for training\n",
+        "    gradient_accumulation_steps=2, # Number of updates steps to accumulate before performing a backward/update pass\n",
+        "    optim=\"paged_adamw_8bit\", # Optimizer to use\n",
+        "    save_steps=100, # Save checkpoint every X updates steps\n",
+        "    logging_steps=10, # Log every X updates steps\n",
+        "    learning_rate=2e-4, # Initial learning rate for AdamW optimizer\n",
+        "    weight_decay=0.001, # Weight decay for AdamW\n",
+        "    fp16=True, # Enable mixed precision training\n",
+        "    bf16=False, # Disable BF16 if using FP16\n",
+        "    max_grad_norm=0.3, # Max gradient norm\n",
+        "    max_steps=-1, # Don't limit training by steps, use epochs\n",
+        "    warmup_ratio=0.03, # Ratio of total steps for a linear warmup from 0 to learning_rate\n",
+        "    # group_by_length=True, # Group sequences of roughly the same length together to speed up training\n",
+        "    lr_scheduler_type=\"constant\", # Learning rate scheduler type\n",
+        "    report_to=\"none\" # Disable reporting to any tracking service\n",
+        ")\n",
+        "\n",
+        "# --- 5. Initialize and Run SFTTrainer ---\n",
+        "\n",
+        "trainer = SFTTrainer(\n",
+        "    model=tuned_model,          # plain (non-PEFT) base model\n",
+        "    train_dataset=dataset,\n",
+        "    peft_config=lora_config,    # SFTTrainer applies LoRA internally\n",
+        "    # dataset_text_field=\"text\", # Name of the column containing the text data\n",
+        "    # tokenizer=tokenizer,\n",
+        "    args=training_args,\n",
+        "    # packing=False, # Whether to pack multiple short examples into one longer sequence to improve efficiency\n",
+        "    # max_seq_length=512, # Max sequence length to use for training\n",
+        ")\n",
+        "\n",
+        "print(\"Starting finetuning...\")\n",
+        "trainer.train()\n",
+        "print(\"Finetuning complete!\")\n",
+        "\n",
+        "# --- 6. (Optional) Save the finetuned model ---\n",
+        "# trainer.save_model(\"./gemma_finetuned_model\")\n",
+        "\n",
+        "# --- 7. (Optional) Merge LoRA adapters with the base model for inference ---\n",
+        "# from peft import AutoPeftModelForCausalLM\n",
+        "# merged_model = AutoPeftModelForCausalLM.from_pretrained(\n",
+        "#     \"./gemma_finetuned_model\",\n",
+        "#     device_map=\"auto\",\n",
+        "#     torch_dtype=torch.bfloat16 # or torch.float16 depending on your hardware\n",
+        "# )\n",
+        "# merged_model.save_pretrained(\"gemma_merged_model\", safe_serialization=True)\n",
+        "# tokenizer.save_pretrained(\"gemma_merged_model\")\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "trusted": true
+      },
+      "source": [
+        "# Duplicate cell removed — run the finetuning cell above.\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "source": [
+        "# Alternative (pick ONE approach, not both):\n",
+        "#\n",
+        "# Option A — let SFTTrainer apply LoRA (used in the cell above):\n",
+        "#   tuned_model = prepare_model_for_kbit_training(tuned_model)\n",
+        "#   trainer = SFTTrainer(model=tuned_model, peft_config=lora_config, ...)\n",
+        "#\n",
+        "# Option B — wrap manually, omit peft_config from SFTTrainer:\n",
+        "#   tuned_model = get_peft_model(tuned_model, lora_config)\n",
+        "#   trainer = SFTTrainer(model=tuned_model, ...)  # no peft_config\n",
+        "\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "machine_shape": "hm"
+    },
+    "kaggle": {
+      "accelerator": "gpu"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.12.13"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 4
+}

scripts/download_model.py

@@ -1,45 +1,104 @@
 #!/usr/bin/env python3
-"""Download the configured GGUF model from Hugging Face Hub for offline dev."""
+"""Download a configured model preset from Hugging Face Hub for offline dev."""
 
 from __future__ import annotations
 
 import argparse
-import os
 from pathlib import Path
 
-from huggingface_hub import hf_hub_download
+from huggingface_hub import hf_hub_download, snapshot_download
+
+from inference.config import get_app_config, get_model_config
+
+
+def _is_local_path(value: str) -> bool:
+    return value.startswith(("./", "../")) or Path(value).is_absolute()
+
+
+def _download_llama_cpp(model, preset_key: str, output_dir: Path) -> Path:
+    if model.model_path:
+        path = Path(model.model_path)
+        if not path.exists():
+            raise SystemExit(f"Local MODEL_PATH does not exist: {model.model_path}")
+        print(f"Preset {preset_key!r} already points to local file: {path}")
+        print(f"Set MODEL_PATH={path} or update models.yaml model_path to use it directly.")
+        return path.resolve()
+
+    if not model.model_repo or not model.model_file:
+        raise SystemExit(f"Preset {preset_key!r} is missing model_repo/model_file.")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = Path(
+        hf_hub_download(
+            repo_id=model.model_repo,
+            filename=model.model_file,
+            local_dir=output_dir,
+            local_dir_use_symlinks=False,
+        )
+    )
+    print(f"Preset: {preset_key} ({model.label})")
+    print(f"Model ready at: {path}")
+    print("Add to models.yaml under that preset:")
+    print(f"  model_path: {path.resolve()}")
+    return path.resolve()
+
+
+def _download_transformers(model, preset_key: str, output_dir: Path) -> Path:
+    if not model.model_id:
+        raise SystemExit(f"Preset {preset_key!r} is missing model_id.")
+
+    if _is_local_path(model.model_id):
+        path = Path(model.model_id).resolve()
+        if not path.exists():
+            raise SystemExit(f"Local model_id does not exist: {model.model_id}")
+        print(f"Preset {preset_key!r} already points to local path: {path}")
+        print(f"Set MODEL_ID={path} or update models.yaml model_id to use it directly.")
+        return path
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    dest = output_dir / model.model_id
+    path = Path(
+        snapshot_download(
+            repo_id=model.model_id,
+            local_dir=dest,
+            local_dir_use_symlinks=False,
+        )
+    )
+    print(f"Preset: {preset_key} ({model.label})")
+    print(f"Model ready at: {path}")
+    print("Add to models.yaml under that preset:")
+    print(f"  model_id: ./{path.relative_to(Path.cwd())}")
+    return path.resolve()
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
-        "--repo",
-        default=os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF"),
-        help="Hugging Face repo containing the GGUF file",
-    )
-    parser.add_argument(
-        "--file",
-        default=os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf"),
-        help="GGUF filename inside the repo",
+        "--preset",
+        default=None,
+        help="Preset key from models.yaml (default: ACTIVE_MODEL or app default)",
     )
     parser.add_argument(
         "--output-dir",
         type=Path,
         default=Path("models"),
-        help="Directory to copy/symlink the downloaded model into",
+        help="Directory to download the model into",
     )
     args = parser.parse_args()
 
-    args.output_dir.mkdir(parents=True, exist_ok=True)
+    app_config = get_app_config()
+    preset_key = args.preset or app_config.active_model
+    model = get_model_config(preset_key)
 
-    path = hf_hub_download(
-        repo_id=args.repo,
-        filename=args.file,
-        local_dir=args.output_dir,
-        local_dir_use_symlinks=False,
-    )
-    print(f"Model ready at: {path}")
-    print(f"Set MODEL_PATH={path} to use this file directly.")
+    if model.backend == "llama_cpp":
+        _download_llama_cpp(model, preset_key, args.output_dir)
+        return
+
+    if model.backend == "transformers":
+        _download_transformers(model, preset_key, args.output_dir)
+        return
+
+    raise SystemExit(f"Preset {preset_key!r} uses unsupported backend {model.backend!r}.")
 
 
 if __name__ == "__main__":

uv.lock

@@ -198,7 +198,7 @@ name = "cuda-bindings"
 version = "13.3.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cuda-pathfinder" },
+    { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" },
@@ -229,34 +229,34 @@ wheels = [
 
 [package.optional-dependencies]
 cudart = [
-    { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" },
 ]
 cufft = [
-    { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cufft", marker = "sys_platform == 'linux'" },
 ]
 cufile = [
     { name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
 ]
 cupti = [
-    { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" },
 ]
 curand = [
-    { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-curand", marker = "sys_platform == 'linux'" },
 ]
 cusolver = [
-    { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" },
 ]
 cusparse = [
-    { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" },
 ]
 nvjitlink = [
-    { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" },
 ]
 nvrtc = [
-    { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" },
 ]
 nvtx = [
-    { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
+    { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" },
 ]
 
 [[package]]
@@ -498,26 +498,25 @@ name = "inference"
 version = "0.1.0"
 source = { editable = "libs/inference" }
 dependencies = [
+    { name = "accelerate" },
     { name = "huggingface-hub" },
     { name = "llama-cpp-python" },
-]
-
-[package.optional-dependencies]
-transformers = [
-    { name = "accelerate" },
+    { name = "pyyaml" },
     { name = "torch" },
+    { name = "torchvision" },
     { name = "transformers" },
 ]
 
 [package.metadata]
 requires-dist = [
-    { name = "accelerate", marker = "extra == 'transformers'", specifier = ">=1.2.0" },
+    { name = "accelerate", specifier = ">=1.2.0" },
     { name = "huggingface-hub", specifier = ">=0.27.0" },
     { name = "llama-cpp-python", specifier = ">=0.3.0" },
-    { name = "torch", marker = "extra == 'transformers'", specifier = ">=2.5.0" },
-    { name = "transformers", marker = "extra == 'transformers'", specifier = ">=4.47.0" },
+    { name = "pyyaml", specifier = ">=6.0.2" },
+    { name = "torch", specifier = ">=2.5.0" },
+    { name = "torchvision", specifier = ">=0.20.0" },
+    { name = "transformers", specifier = ">=5.7.0" },
 ]
-provides-extras = ["transformers"]
 
 [[package]]
 name = "iniconfig"
@@ -720,7 +719,7 @@ name = "nvidia-cublas"
 version = "13.1.1.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cuda-nvrtc" },
+    { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
@@ -759,7 +758,7 @@ name = "nvidia-cudnn-cu13"
 version = "9.20.0.48"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas" },
+    { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" },
@@ -771,7 +770,7 @@ name = "nvidia-cufft"
 version = "12.0.0.61"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink" },
+    { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -801,9 +800,9 @@ name = "nvidia-cusolver"
 version = "12.0.4.66"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas" },
-    { name = "nvidia-cusparse" },
-    { name = "nvidia-nvjitlink" },
+    { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+    { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+    { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -815,7 +814,7 @@ name = "nvidia-cusparse"
 version = "12.6.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink" },
+    { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -1614,6 +1613,38 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/21/afadd25ecd81b3cea1e11c73cf1ab41a983a50271548c3ec7ec3b9efc3e9/torch-2.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f96b63f8287f66a005dd1b5a6abba2920f11156c5e5c4d815f3e2050fd1aa16", size = 123231092, upload-time = "2026-05-13T14:51:18.854Z" },
 ]
 
+[[package]]
+name = "torchvision"
+version = "0.27.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+    { name = "pillow" },
+    { name = "torch" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9e/c8/5cd91932f7f3671b0743dc4ae1a4c16b1d0b45bf4087976277d325bda718/torchvision-0.27.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:1a6dd742a150645126df9e0b2e449874c1d635897c773b322c2e067e98382dfe", size = 1758824, upload-time = "2026-05-13T14:57:15.227Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/36/7fb7d19477b3d93283b52fea11fa8ee30ab9064a08c97b4a6b91445e26cb/torchvision-0.27.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65772ff3ec4f4f5d680e30019835555dd239e7fefee4b0a846375fe1cb1592ef", size = 7831034, upload-time = "2026-05-13T14:57:06.483Z" },
+    { url = "https://files.pythonhosted.org/packages/62/43/dfd894c3f8b01b5b33fde990f0159c1926ebc7b6e2c4193e2efb7da3c4cb/torchvision-0.27.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7a9966a088d06b4cf6c610e03be62de469efa6f2cd2e7c7eed8e925ed6af59ac", size = 7579774, upload-time = "2026-05-13T14:56:59.337Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0c/722e989f9cf026e97ef7cb24a9bb1859e099f72d247ae35388fb89729f73/torchvision-0.27.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c037709072ca9b19750c0cbe9e8bb6f91c9a1be1befa26df33e281deccbd8c7", size = 4021073, upload-time = "2026-05-13T14:57:00.848Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/ae/36547812e6e047c1d80bcacd1b17a340612b08a6e876e0aabf3d0b9228b0/torchvision-0.27.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:41d6dae73e1af09fa82ded597ae57f2a2314285acde54b25890a8f8e51b999d7", size = 1758826, upload-time = "2026-05-13T14:57:05.262Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/30/32c4ea842738728a14e3df8c576c62dedcf5ae5cb6a5c984c6429ebe7524/torchvision-0.27.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:70f071c6f74b60d5fe8851636d8d4cd5f4fa29d57fd9348a87a6f17b990b95ba", size = 7789501, upload-time = "2026-05-13T14:56:57.786Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/24/4d0d48684251bd0673f87d633d5d88ab00227983b00591156eed2f86c8d5/torchvision-0.27.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:aaafa6962c9d91f42503de1957d6fa349907d028c06f335bd95da7a5bc57147d", size = 7579868, upload-time = "2026-05-13T14:56:41.618Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/da/e6edd051d2ba25adf23b120fa97f458dff888d098c51e84724f17d2d1470/torchvision-0.27.0-cp313-cp313-win_amd64.whl", hash = "sha256:aee384a2782c89517c4ab9061d2720ba59fd2ffe5ef89d0a149cc2d43abdf521", size = 4092700, upload-time = "2026-05-13T14:57:09.729Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/23/95dfa40431360f42ca949bf861434bed51164adfa8fb9801e05bf3194f50/torchvision-0.27.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:c5121f1b9ab09a7f73e837871deb8321551f7eaeb19d87aa00de9191968eae44", size = 1845008, upload-time = "2026-05-13T14:57:03.768Z" },
+    { url = "https://files.pythonhosted.org/packages/23/b9/9dbdf76b2b49a75ba8088df6f7c755bdb520afb6c6dbac0102b46cde5e99/torchvision-0.27.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:1c01f0d1091ae22b9dfc082b0a0fe5faaf053686a29b4fb082ba7691375c73cf", size = 7791430, upload-time = "2026-05-13T14:56:56.206Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/6a/e4a16cf2f3310c2ea7760dc5d9054496844391e0f4c1fae87fefac2f3d9e/torchvision-0.27.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dadea3c5ecfd05bbb2a3312ab0374f213c58bf6459cb059122e2f4dfe13d10ed", size = 7668441, upload-time = "2026-05-13T14:57:02.127Z" },
+    { url = "https://files.pythonhosted.org/packages/00/70/01b6461117a6a94b5af3f8ee166bb0f045056f3cf187750c110dabfdfffa/torchvision-0.27.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a49e55055a39a8506fe7e59850522cab004efb2c3839f6057658889c1d69c815", size = 4141602, upload-time = "2026-05-13T14:56:53.449Z" },
+    { url = "https://files.pythonhosted.org/packages/92/22/c0633677b3b3f3e69554a21ac087bf705f829c40cd5e3783507b8c006681/torchvision-0.27.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:c1fac0fc2a7adf29481fc1938a0e7845c57ba1147a986784109c4d98f434ea8c", size = 1758818, upload-time = "2026-05-13T14:56:54.988Z" },
+    { url = "https://files.pythonhosted.org/packages/48/e8/55f9d9667b56dae470e69e31beac9b00d458ea393feec1aae95cc4f3f1c9/torchvision-0.27.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:cbf89764fc76f3f17fbf80c12d5a89c691e91cb9d82c38412aaf0568655ffb19", size = 7789667, upload-time = "2026-05-13T14:56:48.858Z" },
+    { url = "https://files.pythonhosted.org/packages/00/bc/6f8681daf3bbc4c315bb0005110f99d28e3ecd675bf9c8f2c0d393fbac7a/torchvision-0.27.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:91f61b9865423037c327eb56afa207cc72de874e458c361840db9dcf5ce0c0eb", size = 7579848, upload-time = "2026-05-13T14:56:38.209Z" },
+    { url = "https://files.pythonhosted.org/packages/19/6c/8d8020e6bd1e46c53e487c9c4e9457a07f2ee28931028fb5d71e2da40adc/torchvision-0.27.0-cp314-cp314-win_amd64.whl", hash = "sha256:5bb82fc3c55daf1788621e504310b0a286f1069627a8742f692aebb075ef25a7", size = 4119284, upload-time = "2026-05-13T14:56:46.625Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/7e/e78c48662a8d551606efdbe11c6b9c1d6d2391b92cd0e4591b9e6a2412b8/torchvision-0.27.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:2c4099a15150143b9b034730b404a56d572efe0b79489b4c765d929cb4eac7f3", size = 1758828, upload-time = "2026-05-13T14:56:52.293Z" },
+    { url = "https://files.pythonhosted.org/packages/21/dd/d03ee9f9ee7bf11a8c7c776fb8e7fd6102f59c013791a2a4e5175bd6cba7/torchvision-0.27.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b4c6bb0a670dcba017b3643e21902c9b8a1cc1c127d602f1488fa29ec3c6e865", size = 7790618, upload-time = "2026-05-13T14:56:44.721Z" },
+    { url = "https://files.pythonhosted.org/packages/39/08/4002336a74742be70728603ec1769feb2b55e0d19c532c9ec9f92008de76/torchvision-0.27.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1c2db4bde82bc48ebff73436a6adf34d4f809448268a70d9a1285f5c8f92313d", size = 7580217, upload-time = "2026-05-13T14:56:43.274Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/cb/4dd4783eb3565f526ba6e64b6f6ca26c00eacc924cdfe60455db9d91b84b/torchvision-0.27.0-cp314-cp314t-win_amd64.whl", hash = "sha256:72bf547e58ddb948689734eed6f4b6a2031f979dba4fb08e3690688b392e929f", size = 4226392, upload-time = "2026-05-13T14:56:40.235Z" },
+]
+
 [[package]]
 name = "tqdm"
 version = "4.68.1"