diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..45d8057ff14ab50a64772f92b90827c8e9f3c2f8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test-quality: + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install -r requirements.txt -r requirements-dev.txt + + - name: Tests, coverage, lint, type, security + shell: pwsh + run: .\scripts\run_quality.ps1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2583a7e385ed5c479460b7cfde25b0c98c3b6ac8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ +.gradio/ +node_modules/ +#test-results/ +#playwright-report/ +blob-report/ +#assets/e2e/playwright-report/ +#assets/e2e/plant-playwright-report/ +gradio.out.log +gradio.err.log +data/* +!data/.gitkeep +exports/* +!exports/.gitkeep +models_cache/ +hf_cache/ +wandb/ +*.gguf +*.safetensors +*.bin +*.ckpt +*.onnx +*.tflite +*.pt +*.pth +*.model +.env +.env.* +*.pem +*.key diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..789e652c779c1d0c0fa821091d151c430b4271f3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,62 @@ +# AGENTS.md + +## Project Goal +Build a Gradio-based OpenBMB Local AI Workbench for the Build Small Hackathon. +The app should run locally first, deploy as a Hugging Face Space, and document the +small-model choices clearly. + +## Project Memory +- Start with `docs/README.md`. +- Use `docs/TASKS.md` as the task checklist. +- Use `docs/IMPLEMENTATION_STATUS.md` to decide whether something is really done. +- Use `docs/PRD_IMPLEMENTATION_MATRIX.md` before claiming PRD or extension PRD completion. +- Use `docs/ACCEPTANCE_CRITERIA.md` before marking work complete. +- Use `docs/ROADMAP.md` to prioritize hackathon work over stretch work. +- Use `docs/ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md` for hard judge-oriented prioritization. +- Use `docs/TEMPLATE_HOWTO.md` before creating a new domain app from the template. +- Use `docs/PLANT_DISCOVERY_APP_PLAN.md` before changing the Plant Discovery reference app. +- Use `docs/PLANT_MODEL_AND_TRAINING_HOWTO.md` before changing Plant Discovery model or training behavior. +- Use `docs/ARCHITECTURE.md` before changing app structure. +- Use `docs/EXTENDING.md` before adding models, tabs, services, or training. +- Main PRD: `HF_PRD_v1.md`. +- Extension PRD: `HF_PRD_ext.md`. + +## Hackathon Constraints +- Use Gradio as the app surface. +- Keep all model choices at or below 32B parameters. +- Prefer local/offline inference paths when practical. +- Avoid cloud APIs unless the user explicitly approves them. +- Make the README useful for judges: goal, setup, model choice, local run, Space deployment, demo flow. + +## Current MVP Scope +- Start with a working Gradio app and deterministic placeholder service. +- Add real inference incrementally through Ollama, llama.cpp, Transformers, or SGLang. +- Keep training, GGUF export, Trackio, MCP, and agent functionality as documented extension points until implemented. +- Never claim the full PRD is done while placeholders or unchecked PRD tasks remain. + +## Commands +- Create venv: `python -m venv .venv` +- Activate venv: `.venv\Scripts\Activate.ps1` +- Install: `python -m pip install -r requirements.txt` +- Install dev tools: `python -m pip install -r requirements-dev.txt` +- Run locally: `python app.py` +- Run tests: `.\scripts\run_tests.ps1` +- Run performance tests: `.\scripts\run_performance.ps1` +- Run quality checks: `.\scripts\run_quality.ps1` + +## Code Style +- Prefer small modules with clear responsibilities. +- Keep model IDs in `config/models.yaml`; avoid hardcoding them in UI logic. +- Keep user-facing text concise and demo-oriented. +- Do not download model weights automatically on app startup. + +## Verification +- Before claiming the app runs, verify `python app.py` starts locally. +- If Python or dependencies are missing, say exactly what is missing. +- Keep generated caches, model weights, exports, and virtualenvs out of git. +- After implementing a task, update `docs/TASKS.md` and `docs/IMPLEMENTATION_STATUS.md`. +- Every feature needs tests. Add or update unit tests for logic and user-story tests for workflows. +- Whenever a bug, failed check, or regression is found, create or update a test that would catch it before fixing the code. +- Do not mark a feature done until its tests pass or the test blocker is documented. +- Avoid blanket coverage escapes. Do not add `pragma: no cover` without a specific documented reason. +- For PRD work, update `docs/PRD_IMPLEMENTATION_MATRIX.md` along with task/status docs. diff --git a/HF_PRD_ext.md b/HF_PRD_ext.md new file mode 100644 index 0000000000000000000000000000000000000000..89d7c0c03ca007cb797af4a82b83886215514133 --- /dev/null +++ b/HF_PRD_ext.md @@ -0,0 +1,1631 @@ +# OpenBMB Workbench — Planned Extensions: Detailed Spec +> 2026-06-05 | Companion to PRD v2.0 + +--- + +## Table of Contents + +1. [vLLM Serving Tab](#1-vllm-serving-tab) +2. [Ollama Quick-Start](#2-ollama-quick-start) +3. [Reward Model Eval](#3-reward-model-eval) +4. [Synthetic Data Gen](#4-synthetic-data-gen) +5. [Paper-to-Code Agent](#5-paper-to-code-agent) +6. [HF Spaces Deploy](#6-hf-spaces-deploy) +7. [VINDEX Integration](#7-vindex-integration) +8. [OCR Pipeline Hook](#8-ocr-pipeline-hook) +9. [MiniCPM Desk-Pet](#9-minicpm-desk-pet) +10. [MiniCPM-o Audio Tab](#10-minicpm-o-audio-tab) +11. [Cross-Extension Wiring](#11-cross-extension-wiring) + +--- + +## 1. vLLM Serving Tab + +### What it is + +vLLM is a production-grade inference engine built around *PagedAttention* — a KV-cache management +algorithm that treats GPU memory like virtual memory pages. The result is dramatically higher +throughput when multiple requests run concurrently, compared to naive Transformers inference. + +In the workbench context, vLLM adds a fourth inference mode alongside llama.cpp, SGLang, and +Ollama. You use it when you want OpenAI-compatible HTTP endpoints, continuous batching, or when +benchmarking production serving latency. + +### Why it matters + +| Scenario | Benefit | +|----------|---------| +| Benchmarking fine-tuned LoRA | Compare throughput before/after fine-tune | +| Multi-user demo | Queue and batch concurrent requests | +| Production deployment | OpenAI-compatible API, drop-in for existing tooling | +| MiniCPM4.1-8B long context | PagedAttention shines on 128K context — avoids OOM | + +### Architecture + +``` +models/vllm_runner.py + VLLMRunner + .start(model_id, cfg) → subprocess: vllm serve ... + .stop() → terminate subprocess + .chat(messages) → str → POST /v1/chat/completions + .batch(prompts) → list[str]→ concurrent POST via asyncio + .stats() → dict → GET /metrics (Prometheus) +``` + +```python +# models/vllm_runner.py +import subprocess, asyncio, requests +from openai import AsyncOpenAI + +class VLLMRunner: + def __init__(self, cfg: dict): + self.model_id = cfg["hf_id"] + self.port = cfg.get("port", 8000) + self.gpu_memory = cfg.get("gpu_memory_utilization", 0.85) + self.trust_rc = cfg.get("trust_remote_code", False) + self._proc = None + self._client = AsyncOpenAI( + base_url=f"http://localhost:{self.port}/v1", + api_key="vllm-local" + ) + + def start(self): + cmd = [ + "vllm", "serve", self.model_id, + "--port", str(self.port), + "--gpu-memory-utilization", str(self.gpu_memory), + ] + if self.trust_rc: + cmd.append("--trust-remote-code") + self._proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + self._wait_ready() + + def _wait_ready(self, timeout: int = 60): + import time + for _ in range(timeout): + try: + r = requests.get(f"http://localhost:{self.port}/health") + if r.status_code == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("vLLM server did not become healthy") + + async def chat(self, messages: list[dict], **kwargs) -> str: + resp = await self._client.chat.completions.create( + model=self.model_id, + messages=messages, + max_tokens=kwargs.get("max_tokens", 512), + ) + return resp.choices[0].message.content + + async def batch(self, prompts: list[str], system: str = "") -> list[str]: + tasks = [ + self.chat([ + {"role": "system", "content": system}, + {"role": "user", "content": p}, + ]) + for p in prompts + ] + return await asyncio.gather(*tasks) + + def stats(self) -> dict: + """Prometheus metrics parsed to dict. Returns throughput + latency.""" + r = requests.get(f"http://localhost:{self.port}/metrics") + lines = r.text.splitlines() + metrics = {} + for line in lines: + if line.startswith("vllm:") and not line.startswith("#"): + key, val = line.rsplit(" ", 1) + metrics[key] = float(val) + return metrics + + def stop(self): + if self._proc: + self._proc.terminate() + self._proc = None +``` + +### UI tab (models_tab.py → extend) + +Add a "vLLM" section to the models tab: + +```python +with gr.Tab("⚡ vLLM"): + model_dd = gr.Dropdown(label="Model", choices=list_vllm_compatible()) + gpu_mem_sl = gr.Slider(0.5, 0.95, value=0.85, label="GPU memory utilization") + start_btn = gr.Button("Start server") + stop_btn = gr.Button("Stop server") + status_lbl = gr.Label(label="Status") + metrics_json = gr.JSON(label="Live metrics") + + def start_vllm(model_id, gpu_mem): + cfg = {**model_registry.get(model_id).cfg, + "gpu_memory_utilization": gpu_mem} + runner = VLLMRunner(cfg) + runner.start() + model_registry.register(f"{model_id}_vllm", runner) + return "Running", runner.stats() + + start_btn.click(start_vllm, [model_dd, gpu_mem_sl], [status_lbl, metrics_json]) +``` + +### Trackio events fired + +```python +trackio.init(project="workbench", run_name="vllm_benchmark") +trackio.log({"throughput_tok_per_s": ..., "p50_latency_ms": ..., "gpu_mem_used": ...}) +trackio.finish() +``` + +--- + +## 2. Ollama Quick-Start + +### What it is + +Ollama is zero-configuration local model serving. One `ollama pull` downloads a quantized model +and one `ollama serve` runs it. No CUDA setup, no Python environment issues. The REST API is +OpenAI-compatible on port 11434. + +MiniCPM-V-4.6 and MiniCPM5-1B are both in the Ollama registry: +```bash +ollama pull openbmb/minicpm-v4.6 +ollama pull openbmb/minicpm5-1b +``` + +### Why it matters + +Ollama is the fastest path from "nothing" to "running model" — ideal for demos, non-GPU machines +(Apple Silicon is well-optimized), and users who shouldn't need to understand quantization. + +### Architecture + +``` +models/ollama_runner.py + OllamaRunner + .pull(model_id) → subprocess: ollama pull ... + .chat(messages) → str → POST http://localhost:11434/api/chat + .generate(prompt) → str → POST http://localhost:11434/api/generate (streaming) + .list() → list[str] → GET /api/tags +``` + +```python +# models/ollama_runner.py +import requests, subprocess, json +from typing import Generator + +class OllamaRunner: + BASE = "http://localhost:11434" + + def __init__(self, model_id: str): + # Ollama uses "openbmb/minicpm-v4.6" style IDs directly + self.model_id = model_id + + @staticmethod + def pull(model_id: str): + subprocess.run(["ollama", "pull", model_id], check=True) + + @staticmethod + def list_local() -> list[str]: + r = requests.get(f"{OllamaRunner.BASE}/api/tags") + return [m["name"] for m in r.json().get("models", [])] + + def chat(self, messages: list[dict], stream: bool = False) -> str | Generator: + payload = {"model": self.model_id, "messages": messages, "stream": stream} + r = requests.post(f"{self.BASE}/api/chat", json=payload, stream=stream) + if not stream: + return r.json()["message"]["content"] + # Generator for Gradio streaming + def _stream(): + for line in r.iter_lines(): + if line: + chunk = json.loads(line) + yield chunk["message"]["content"] + if chunk.get("done"): + break + return _stream() + + def vision_chat(self, image_b64: str, prompt: str) -> str: + """Multimodal chat — Ollama passes images as base64 in the message.""" + messages = [{ + "role": "user", + "content": prompt, + "images": [image_b64], + }] + return self.chat(messages) +``` + +### UI integration + +The models tab gets an "Ollama" subtab with a model browser, pull button, and instant chat +that requires zero setup — the simplest entry point to the whole workbench. + +```python +with gr.Tab("🦙 Ollama"): + available = gr.Dropdown(label="Pull model", + choices=["openbmb/minicpm-v4.6", "openbmb/minicpm5-1b"], + allow_custom_value=True) + pull_btn = gr.Button("Pull") + local_list = gr.JSON(label="Locally available") + pull_status= gr.Textbox(label="Status") + + pull_btn.click( + lambda m: (OllamaRunner.pull(m), OllamaRunner.list_local()), + [available], + [pull_status, local_list] + ) +``` + +--- + +## 3. Reward Model Eval + +### What it is + +A reward model is a model trained to score (prompt, response) pairs — answering "how good is this +output?" It's the missing piece between fine-tuning and verified alignment improvement. Without it +you can train a LoRA and only know quantitatively that loss went down, not whether outputs actually +got better by human-relevant criteria. + +### Why it matters + +- Validates that LoRA fine-tuning improved quality (not just minimized loss) +- Enables best-of-N sampling: generate N responses, keep highest-scored +- Enables DPO data creation: generate response pairs, reward model labels preferences +- Closes the RLHF loop within the workbench itself + +### Reward model options + +| Model | Size | Focus | +|-------|------|-------| +| `OpenAssistant/reward-model-deberta-v3-large-v2` | 450M | General helpfulness | +| `Salesforce/SFR-Reward-FsfairX-LLaMA3-RM-v0.1` | 8B | Instruction following | +| MiniCPM5-1B itself (self-eval) | 1B | Domain-specific, via prompt | + +For the workbench, using MiniCPM5-1B as a judge (LLM-as-judge pattern) is the lowest-friction +option since the model is already loaded. + +### Architecture + +``` +training/reward_eval.py + RewardEvaluator + .score(prompt, response) → float + .best_of_n(prompt, n, generator) → str + .create_dpo_pairs(dataset, generator, n=4) → Dataset + .eval_lora_vs_base(base_svc, lora_svc, eval_ds) → dict +``` + +```python +# training/reward_eval.py +from transformers import pipeline, AutoModelForSequenceClassification, AutoTokenizer +import torch + +class RewardEvaluator: + def __init__(self, reward_model_id: str = "OpenAssistant/reward-model-deberta-v3-large-v2"): + self.pipe = pipeline( + "text-classification", + model=reward_model_id, + device=0 if torch.cuda.is_available() else -1, + ) + + def score(self, prompt: str, response: str) -> float: + """Returns a scalar reward score (higher = better).""" + text = f"Human: {prompt}\n\nAssistant: {response}" + result = self.pipe(text, truncation=True, max_length=512) + return result[0]["score"] + + def best_of_n( + self, + prompt: str, + n: int, + generator_fn, # callable: prompt -> str + ) -> tuple[str, float]: + """Generate N responses, return the one with highest reward.""" + candidates = [(generator_fn(prompt), ) for _ in range(n)] + scored = [(r[0], self.score(prompt, r[0])) for r in candidates] + return max(scored, key=lambda x: x[1]) + + def create_dpo_pairs( + self, + prompts: list[str], + generator_fn, + n: int = 4, + ): + """ + For each prompt: generate n responses, rank by reward, + return (prompt, chosen, rejected) triplets for DPO training. + """ + from datasets import Dataset + rows = [] + for prompt in prompts: + responses = [(generator_fn(prompt), ) for _ in range(n)] + scored = sorted( + [(r[0], self.score(prompt, r[0])) for r in responses], + key=lambda x: x[1], reverse=True + ) + if len(scored) >= 2: + rows.append({ + "prompt": prompt, + "chosen": scored[0][0], + "rejected": scored[-1][0], + "reward_gap": scored[0][1] - scored[-1][1], + }) + return Dataset.from_list(rows) + + def eval_lora_vs_base( + self, + base_svc, + lora_svc, + eval_prompts: list[str], + ) -> dict: + """ + Compare base vs LoRA checkpoint by average reward score. + Returns win rate and per-prompt scores. + """ + base_scores = [self.score(p, base_svc.generate(p)) for p in eval_prompts] + lora_scores = [self.score(p, lora_svc.generate(p)) for p in eval_prompts] + wins = sum(l > b for l, b in zip(lora_scores, base_scores)) + return { + "base_mean": sum(base_scores) / len(base_scores), + "lora_mean": sum(lora_scores) / len(lora_scores), + "lora_win_rate": wins / len(eval_prompts), + "per_prompt": list(zip(eval_prompts, base_scores, lora_scores)), + } +``` + +### Trackio logging + +```python +results = evaluator.eval_lora_vs_base(base_svc, lora_svc, eval_prompts) +trackio.init(project="workbench", run_name="reward_eval") +trackio.log({ + "base_reward_mean": results["base_mean"], + "lora_reward_mean": results["lora_mean"], + "lora_win_rate": results["lora_win_rate"], +}) +trackio.finish() +``` + +--- + +## 4. Synthetic Data Gen + +### What it is + +The ml-intern finding: *when real data is insufficient, have an LLM generate training data*. +This module does exactly that — it uses a capable model (MiniCPM4.1-8B or a cloud model via +HF Router) to generate diverse, high-quality (prompt, response) pairs on demand. + +### Why it matters + +Real-world fine-tuning is often blocked not by compute but by data. You have 50 good examples +but need 5000. Synthetic gen + quality filtering bridges that gap, especially for specialized +domains (plant species, historical OCR corrections, industrial inspection defect labels). + +### Architecture + +``` +datasets/synthetic.py + SyntheticGenerator + .generate(topic, n, schema) → Dataset + .augment(existing_ds, n) → Dataset + .filter_quality(ds, min_score) → Dataset + .generate_dpo_pairs(topic, n) → Dataset +``` + +```python +# datasets/synthetic.py +import json +from datasets import Dataset + +GENERATION_PROMPT = """You are a training data generator. Generate {n} diverse, high-quality +training examples for the topic: {topic}. + +Output ONLY a valid JSON array. Each item must have these fields: {schema} +No explanation, no markdown, no preamble. Raw JSON array only.""" + +class SyntheticGenerator: + def __init__(self, generator_svc): + """generator_svc: any loaded ModelService with a .generate(prompt) method.""" + self.gen = generator_svc + + def generate( + self, + topic: str, + n: int = 100, + schema: dict | None = None, + ) -> Dataset: + """ + Generate n training examples on a topic. + schema: dict of field_name → description, e.g. + {"instruction": "task to perform", "response": "ideal answer"} + """ + schema = schema or {"instruction": "user task", "response": "ideal answer"} + schema_str = ", ".join(f'"{k}": "{v}"' for k, v in schema.items()) + + # Generate in batches of 20 to stay within context + rows = [] + for batch_start in range(0, n, 20): + batch_n = min(20, n - batch_start) + prompt = GENERATION_PROMPT.format( + n=batch_n, topic=topic, schema="{" + schema_str + "}" + ) + raw = self.gen.generate(prompt) + try: + # Strip any accidental markdown fences + clean = raw.strip().lstrip("```json").lstrip("```").rstrip("```") + batch = json.loads(clean) + rows.extend(batch) + except json.JSONDecodeError: + # Skip malformed batches; log the failure + continue + + return Dataset.from_list(rows) + + def augment(self, existing_ds: Dataset, n: int) -> Dataset: + """ + Use existing examples as few-shot demonstrations to generate n more. + Samples up to 5 examples from existing_ds as context. + """ + import random + samples = existing_ds.shuffle().select(range(min(5, len(existing_ds)))) + examples_str = json.dumps(samples.to_list(), indent=2) + + prompt = f"""Here are {len(samples)} example training items: +{examples_str} + +Generate {n} MORE diverse examples in the exact same JSON format. +Output only the JSON array, no explanation.""" + + raw = self.gen.generate(prompt) + try: + clean = raw.strip().lstrip("```json").rstrip("```") + new_rows = json.loads(clean) + except json.JSONDecodeError: + return existing_ds + return Dataset.from_list(existing_ds.to_list() + new_rows) + + def filter_quality( + self, + ds: Dataset, + reward_evaluator=None, + min_score: float = 0.6, + ) -> Dataset: + """ + Filter with reward model if available, else heuristic filters. + Heuristics: min length, no repeated n-grams, valid JSON fields. + """ + if reward_evaluator: + def _score(row): + return reward_evaluator.score( + row.get("instruction", ""), + row.get("response", "") + ) >= min_score + return ds.filter(_score) + else: + # Basic heuristics + def _heuristic(row): + resp = row.get("response", "") + return ( + len(resp) >= 20 and # not too short + len(resp) <= 4096 and # not too long + resp.count(resp[:20]) < 3 # not repetitive + ) + return ds.filter(_heuristic) + + def generate_for_domain( + self, + domain: str, + output_path: str, + n: int = 500, + ): + """ + Convenience method: generate, augment, filter, save to disk. + Use for plant ID: domain="Plant species identification from photo descriptions" + """ + ds = self.generate(topic=domain, n=n // 2) + ds = self.augment(ds, n=n // 2) + ds = self.filter_quality(ds) + ds.save_to_disk(output_path) + return ds +``` + +### Domain-specific example: plant ID + +```python +gen = SyntheticGenerator(model_registry.get("minicpm41_8b")) +plant_ds = gen.generate_for_domain( + domain="Identifying plant species from visual descriptions. " + "Include common name, latin name, family, key visual features, and care tips.", + output_path="data/synthetic_plants", + n=2000, +) +# → 2000 synthetic (description → species JSON) training pairs +``` + +--- + +## 5. Paper-to-Code Agent + +### What it is + +An autonomous agent that takes an arXiv paper URL or title, reads the methodology section, +and implements the described technique within the workbench codebase. Directly inspired by +the ml-intern architecture (Research → Plan → Implement → Trace). + +### Why it matters + +The gap between reading a paper and running an experiment is usually days of engineering. This +agent compresses that to minutes for techniques that fit the workbench's model family. +Practical use cases: implement a new PEFT variant, add a new evaluation metric, adapt a new +data augmentation from a recent VLM paper. + +### Architecture + +``` +agent/paper_agent.py + PaperAgent + .run(paper_ref) → AgentResult + → Phase 1: Research (fetch + parse paper) + → Phase 2: Plan (identify workbench integration points) + → Phase 3: Implement (generate + write code) + → Phase 4: Test (run + log to Trackio) + → Phase 5: Trace (upload session to HF Dataset) +``` + +```python +# agent/paper_agent.py +import re +from dataclasses import dataclass, field +from huggingface_hub import HfApi +from smolagents import CodeAgent, HfApiModel +import trackio + +@dataclass +class AgentResult: + paper_title: str = "" + summary: str = "" + files_modified: list[str] = field(default_factory=list) + test_results: dict = field(default_factory=dict) + trace_url: str = "" + +class PaperAgent: + SYSTEM_PROMPT = """You are an ML engineer working inside the OpenBMB Workbench codebase. +Given a research paper, your job is to: +1. Understand the core algorithm or technique. +2. Identify which module in the workbench it extends (training/, models/, datasets/, tools/). +3. Implement it as a new class or function, following the existing patterns. +4. Write a simple test that runs within the workbench and logs results to Trackio. + +The workbench uses: transformers, peft, trl, trackio, mcp, gradio. +All new code must: fire events via the EventBus, log to Trackio, register in the Registry.""" + + def __init__(self, orchestrator_model: str = "openbmb/MiniCPM4.1-8B"): + self.model = HfApiModel(orchestrator_model) + self.api = HfApi() + self._log = [] + + def run(self, paper_ref: str) -> AgentResult: + """ + paper_ref: arXiv URL like "https://arxiv.org/abs/2106.09685" + or paper title like "LoRA: Low-Rank Adaptation of Large Language Models" + """ + result = AgentResult() + + # Phase 1: Research + paper_text = self._fetch_paper(paper_ref) + result.paper_title = self._extract_title(paper_text) + + # Phase 2: Plan + plan = self._plan(paper_text) + + # Phase 3: Implement + code_files = self._implement(plan, paper_text) + result.files_modified = list(code_files.keys()) + for path, code in code_files.items(): + self._write_file(path, code) + + # Phase 4: Test + trackio.init(project="workbench", run_name=f"paper_agent_{result.paper_title[:30]}") + test_result = self._test(code_files) + result.test_results = test_result + trackio.log({"test_passed": test_result.get("passed", False), **test_result}) + trackio.finish() + + # Phase 5: Trace (ml-intern pattern) + result.trace_url = self._upload_trace(result) + + return result + + def _fetch_paper(self, paper_ref: str) -> str: + """Fetch paper text via HF Papers API or arXiv.""" + import requests + if "arxiv.org" in paper_ref: + arxiv_id = paper_ref.split("/abs/")[-1] + r = requests.get(f"https://export.arxiv.org/abs/{arxiv_id}") + return r.text + # Fall back to HF Papers search + from huggingface_hub import list_papers + results = list(list_papers(query=paper_ref, limit=1)) + return str(results[0]) if results else "" + + def _plan(self, paper_text: str) -> str: + """Ask the LLM to analyze the paper and produce an integration plan.""" + agent = CodeAgent(tools=[], model=self.model, max_steps=5) + return agent.run( + f"Read this paper excerpt and produce a 5-step integration plan " + f"for the OpenBMB Workbench:\n\n{paper_text[:8000]}" + ) + + def _implement(self, plan: str, paper_text: str) -> dict[str, str]: + """Generate code files from the plan.""" + agent = CodeAgent(tools=[], model=self.model, max_steps=15) + code = agent.run( + f"Implementation plan:\n{plan}\n\n" + f"Paper details:\n{paper_text[:4000]}\n\n" + f"Generate the Python file(s). Return a JSON dict: " + f"{{\"path/to/file.py\": \"file_content\", ...}}" + ) + import json + try: + return json.loads(code) + except Exception: + return {} + + def _write_file(self, path: str, content: str): + import os + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(content) + + def _test(self, code_files: dict) -> dict: + """Run a quick import + instantiation test on generated files.""" + results = {} + for path in code_files: + try: + module_name = path.replace("/", ".").replace(".py", "") + import importlib.util, sys + spec = importlib.util.spec_from_file_location(module_name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + results[path] = "imported_ok" + except Exception as e: + results[path] = f"error: {e}" + results["passed"] = all("ok" in v for v in results.values()) + return results + + def _upload_trace(self, result: AgentResult) -> str: + """Upload session trace to private HF Dataset (ml-intern pattern).""" + import json, datetime + session = { + "timestamp": datetime.datetime.utcnow().isoformat(), + "paper": result.paper_title, + "files_modified": result.files_modified, + "test_results": result.test_results, + } + user = self.api.whoami()["name"] + dataset_id = f"{user}/workbench-paper-sessions" + # Upload as JSONL + # ... (HF Dataset API) + return f"https://huggingface.co/datasets/{dataset_id}" +``` + +--- + +## 6. HF Spaces Deploy + +### What it is + +One-click packaging and deployment of the current workbench state as a Hugging Face Space. +The Space runs the same Gradio app on HF infrastructure, shareable via URL, with ZeroGPU +support for serverless GPU access. + +### Why it matters + +Sharing a workbench state with a collaborator currently requires: git push, environment setup, +model download, config sync. With one-click deploy: copy URL → colleague sees the live app. +For hackathons this is especially valuable — deploy a domain-specific variant as a demo Space +in under 2 minutes. + +### Architecture + +``` +deploy/spaces.py + SpacesDeployer + .prepare_repo() → creates/updates HF Space repo + .upload_code() → pushes app code (not model weights) + .set_hardware(tier) → sets GPU tier in README + .configure_secrets(env_vars) → sets HF Space secrets + .deploy() → trigger Space rebuild + .get_url() → returns live Space URL +``` + +```python +# deploy/spaces.py +import os, shutil, tempfile +from pathlib import Path +from huggingface_hub import HfApi, SpaceHardware + +class SpacesDeployer: + HARDWARE_MAP = { + "cpu": SpaceHardware.CPU_BASIC, + "t4": SpaceHardware.T4_SMALL, + "t4_lg": SpaceHardware.T4_MEDIUM, + "a10": SpaceHardware.A10G_SMALL, + "a100": SpaceHardware.A100_LARGE, + "zero": SpaceHardware.CPU_BASIC, # ZeroGPU: uses CPU_BASIC + @spaces.GPU + } + EXCLUDE = {".git", "__pycache__", "exports", "data", "checkpoints", + "*.gguf", "*.bin", "*.safetensors", ".env"} + + def __init__(self, space_id: str, hardware: str = "zero"): + self.api = HfApi() + self.space_id = space_id # "username/my-workbench" + self.hardware = hardware + + def deploy( + self, + src_dir: str = ".", + env_vars: dict | None = None, + ) -> str: + """Full deploy pipeline. Returns live Space URL.""" + self._create_or_update_repo() + self._upload_code(src_dir) + self._configure_secrets(env_vars or {}) + self._patch_app_for_zerogpu() + return f"https://huggingface.co/spaces/{self.space_id}" + + def _create_or_update_repo(self): + try: + self.api.create_repo( + repo_id=self.space_id, + repo_type="space", + space_sdk="gradio", + private=False, + exist_ok=True, + ) + except Exception as e: + print(f"Repo create/update: {e}") + + def _upload_code(self, src_dir: str): + with tempfile.TemporaryDirectory() as tmp: + src = Path(src_dir) + dest = Path(tmp) + # Copy only non-excluded files + for item in src.rglob("*"): + if any(item.match(pat) for pat in self.EXCLUDE): + continue + rel = item.relative_to(src) + target = dest / rel + target.parent.mkdir(parents=True, exist_ok=True) + if item.is_file(): + shutil.copy2(item, target) + self.api.upload_folder( + folder_path=str(dest), + repo_id=self.space_id, + repo_type="space", + commit_message="workbench deploy", + ) + + def _configure_secrets(self, env_vars: dict): + for key, val in env_vars.items(): + self.api.add_space_secret( + repo_id=self.space_id, + key=key, + value=val, + ) + # Always set hardware + self.api.request_space_hardware( + repo_id=self.space_id, + hardware=self.HARDWARE_MAP[self.hardware], + ) + + def _patch_app_for_zerogpu(self): + """ + If hardware=zero, wrap inference functions with @spaces.GPU decorator. + Uploads a patched app.py. + """ + if self.hardware != "zero": + return + # Read existing app.py from Space, inject @spaces.GPU, re-upload + app_content = self.api.hf_hub_download( + repo_id=self.space_id, filename="app.py", repo_type="space" + ) + with open(app_content) as f: + code = f.read() + # Simple injection: add import and decorator before inference functions + patched = "import spaces\n" + code.replace( + "def run_inference(", + "@spaces.GPU\ndef run_inference(" + ) + self.api.upload_file( + path_or_fileobj=patched.encode(), + path_in_repo="app.py", + repo_id=self.space_id, + repo_type="space", + ) +``` + +### UI: deploy button in any tab header + +```python +with gr.Row(): + space_id_box = gr.Textbox(placeholder="username/my-workbench", label="Space ID") + hardware_dd = gr.Dropdown(["cpu","t4","a10","zero"], value="zero", label="Hardware") + deploy_btn = gr.Button("🚀 Deploy to HF Spaces") + deploy_url = gr.Textbox(label="Live URL", interactive=False) + + def do_deploy(space_id, hw): + d = SpacesDeployer(space_id, hw) + url = d.deploy(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN","")}) + return url + + deploy_btn.click(do_deploy, [space_id_box, hardware_dd], deploy_url) +``` + +--- + +## 7. VINDEX Integration + +### What it is + +VINDEX is your own knowledge-editing engine (ki-fusion-labs.de). It exposes eight methods for +mechanistic interpretability and targeted weight editing on transformer models. Integrating it +into the workbench closes the loop between training a LoRA and verifying (or surgically fixing) +what the model actually knows. + +VINDEX endpoints (from your PRD v1): +- `logit_lens` — per-layer prediction visualization +- `slot_neighbors` — embedding space neighbors +- `layer_contribution` — per-layer influence on final prediction +- `transition_spectrum` — attention head transition matrix +- `calibrated_edit` — targeted fact edit in weight space +- `derive_scale` — scaling factor derivation for safe edits +- `star_spread` — spread edit across semantically related slots +- `protect_relations` — guard against side effects + +### Why it matters for the workbench + +After LoRA fine-tuning a model on plant data: +1. `logit_lens` — does the model correctly predict "Rosa" at the right layer for a rose image? +2. `calibrated_edit` — if it consistently misidentifies Acer as Quercus, edit that slot +3. `star_spread` — propagate the Acer correction to closely related maple species +4. `protect_relations` — verify the edit didn't break "plant → living thing → organism" + +This is not possible with LoRA alone — LoRA changes weight statistics globally. VINDEX does +surgical point edits, making it a *complement* to LoRA, not a replacement. + +### Architecture + +``` +tools/vindex_tool.py + VINDEXClient + .logit_lens(model, tokenizer, text) → dict[layer, prediction] + .calibrated_edit(model, subject, relation, old_obj, new_obj) → model + .star_spread(model, anchor_subject, n=5) → list[affected_slots] + .protect_relations(model, protected_triplets) → model + .layer_contribution(model, text) → dict[layer, score] + .slot_neighbors(model, token_id, n=10) → list[str] +``` + +```python +# tools/vindex_tool.py +""" +VINDEX integration. +Assumes VINDEX FastAPI server is running locally on port 8765, +OR VINDEX modules are importable from your local install. +""" +import requests +from mcp.server.fastmcp import FastMCP + +VINDEX_BASE = "http://localhost:8765" # your local VINDEX FastAPI + +mcp = FastMCP("VINDEXTools") + +@mcp.tool() +async def logit_lens( + model_id: str, + text: str, + layer_range: tuple[int, int] = (0, -1), +) -> dict: + """ + Run logit lens on a loaded model for the given text. + Returns per-layer top-5 token predictions and probabilities. + Useful for finding the 'phase layer' where the model commits to an answer. + """ + r = requests.post(f"{VINDEX_BASE}/logit_lens", json={ + "model_id": model_id, + "text": text, + "layer_range": list(layer_range), + }) + return r.json() + +@mcp.tool() +async def calibrated_edit( + model_id: str, + subject: str, + relation: str, + old_obj: str, + new_obj: str, + causal_window: int = 3, # ±3 layers around logit lens phase layer +) -> dict: + """ + Perform a targeted knowledge edit: change the model's belief about + (subject, relation) from old_obj to new_obj. + causal_window: restrict causal search to ±N layers around phase layer. + Returns edit_success, layers_modified, side_effect_score. + """ + r = requests.post(f"{VINDEX_BASE}/calibrated_edit", json={ + "model_id": model_id, + "subject": subject, + "relation": relation, + "old_obj": old_obj, + "new_obj": new_obj, + "causal_window": causal_window, + }) + return r.json() + +@mcp.tool() +async def star_spread( + model_id: str, + anchor_subject: str, + n_neighbors: int = 5, +) -> dict: + """ + Find semantically related slots and spread a recent edit across them. + Example: after editing "Acer palmatum → maple", also update + "Acer japonicum", "Acer shirasawanum" etc. + Returns list of affected subjects and their edit scores. + """ + r = requests.post(f"{VINDEX_BASE}/star_spread", json={ + "model_id": model_id, + "anchor_subject": anchor_subject, + "n_neighbors": n_neighbors, + }) + return r.json() + +@mcp.tool() +async def protect_relations( + model_id: str, + protected_triplets: list[dict], # [{"s": ..., "r": ..., "o": ...}] +) -> dict: + """ + After a knowledge edit, verify that listed subject-relation-object triplets + remain intact. Returns a pass/fail table and a side_effect_score. + """ + r = requests.post(f"{VINDEX_BASE}/protect_relations", json={ + "model_id": model_id, + "protected_triplets": protected_triplets, + }) + return r.json() + +@mcp.tool() +async def layer_contribution( + model_id: str, + text: str, +) -> dict: + """ + Per-layer contribution score to the final prediction. + Use to find which layers drive the target behavior before editing. + """ + r = requests.post(f"{VINDEX_BASE}/layer_contribution", json={ + "model_id": model_id, + "text": text, + }) + return r.json() +``` + +### Known bugs to fix before integration (from VINDEX PRD v1) + +1. **GPU memory leak** — after repeated edits, VRAM grows unbounded. Fix: explicitly call + `torch.cuda.empty_cache()` after each `calibrated_edit` call and detach gradient graphs. + +2. **Dead-code blocks** — several helper functions in the weight-surgery path are unreachable + after a recent refactor. Before integrating: `grep -n "def " vindex/core.py | xargs` and + verify each function has at least one call site. + +3. **Weight imbalance** — `star_spread` can over-edit related slots if `n_neighbors > 5`. + Hard-cap at 5 in the MCP tool until the scaling formula is validated. + +4. **Forward optimization** — restrict causal search to ±3 layers around the logit lens + phase layer (already implemented as `causal_window` param above). + +### UI tab: "🧠 Knowledge Editor" + +``` +Inputs: + Model selector (loaded models) + Subject text (e.g. "Acer palmatum") + Relation (e.g. "is a type of") + Old object (e.g. "oak") + New object (e.g. "maple") + [Run Logit Lens] button → displays per-layer heatmap via gr.Plot + [Apply Edit] button → runs calibrated_edit + [Spread] button → runs star_spread + [Verify] button → runs protect_relations on a default triplet set + +Outputs: + Per-layer prediction table + Edit success / layers modified + Side effect score (0 = safe, 1 = dangerous) +``` + +--- + +## 8. OCR Pipeline Hook + +### What it is + +Your self-improving multilingual OCR pipeline (Latin, Arabic, Cyrillic) already exists and +produces output files: image + predicted_text + confidence scores. This extension hooks those +outputs directly into the workbench Field Notes system, creating a tight correction loop: + +``` +OCR pipeline outputs (uncertain predictions) + ↓ +Auto-created Field Notes (image + OCR text + empty correction field) + ↓ +Human reviews in UI → fills in correction + ↓ +Accepted corrections auto-tagged "use_for_training=True" + ↓ +LoRA training run on correction pairs + ↓ +Better OCR model → fewer uncertain predictions +``` + +This is the active learning loop your OCR pipeline was designed for but didn't yet have +a clean UI for corrections and retraining. + +### Architecture + +``` +datasets/ocr_loader.py + OCRPipelineLoader + .watch(output_dir, threshold) → poll for new low-confidence outputs + .ingest(output_dir) → batch import all outputs + .to_field_notes(threshold) → FieldNote[] (uncertain ones only) + .to_training_dataset() → Dataset (corrected ones only) +``` + +```python +# datasets/ocr_loader.py +import json, os +from pathlib import Path +from datasets.field_notes import FieldNote, FieldNoteStore +from core.events import bus, EventType, Event + +class OCRPipelineLoader: + """ + Watches a directory written by the OCR pipeline. + Expected format per document: + .json → {"image_path": ..., "predicted_text": ..., + "confidence": float, "script": "latin"|"arabic"|"cyrillic"} + """ + def __init__( + self, + output_dir: str, + store: FieldNoteStore, + confidence_threshold: float = 0.85, + ): + self.output_dir = Path(output_dir) + self.store = store + self.threshold = confidence_threshold + + def ingest(self, limit: int | None = None) -> int: + """ + Read all pipeline outputs. Create Field Notes for uncertain predictions + (confidence < threshold). Skip already-ingested docs. + Returns number of new Field Notes created. + """ + count = 0 + json_files = sorted(self.output_dir.glob("*.json")) + if limit: + json_files = json_files[:limit] + + for jf in json_files: + try: + data = json.loads(jf.read_text()) + except json.JSONDecodeError: + continue + + # Skip high-confidence outputs + if data.get("confidence", 1.0) >= self.threshold: + continue + + note = FieldNote( + id=f"ocr_{jf.stem}", + model_id="ocr_pipeline", + modality="image", + image_path=data["image_path"], + prompt=( + f"Transcribe this {data.get('script','latin')} text accurately. " + f"OCR predicted: '{data['predicted_text']}'" + ), + response=data["predicted_text"], + correction="", # human fills this in + tags=[ + f"script:{data.get('script','unknown')}", + f"conf:{data.get('confidence',0.0):.2f}", + "source:ocr_pipeline", + ], + ) + self.store.save(note) + count += 1 + + # Fire event + import asyncio + asyncio.run(bus.emit(Event( + type=EventType.DATASET_LOADED, + payload={"source": "ocr_pipeline", "new_notes": count} + ))) + return count + + def watch(self, poll_interval: int = 30): + """ + Background thread: poll output_dir every N seconds, ingest new files. + Use in production when OCR pipeline runs continuously. + """ + import threading, time + seen = set() + def _poll(): + while True: + for jf in self.output_dir.glob("*.json"): + if jf.stem not in seen: + seen.add(jf.stem) + self.ingest.__wrapped__([jf]) # single-file ingest + time.sleep(poll_interval) + t = threading.Thread(target=_poll, daemon=True) + t.start() + + def to_training_dataset(self, script_filter: str | None = None): + """ + Export corrected field notes as a training dataset. + schema: {"image_path": ..., "instruction": ..., "response": ...} + Ready to pass to LoRATextTrainer or a vision LoRA config. + """ + from datasets import Dataset + query = "SELECT data FROM notes WHERE json_extract(data,'$.correction') != ''" + if script_filter: + query += f" AND json_extract(data,'$.tags') LIKE '%script:{script_filter}%'" + + rows = [ + json.loads(r[0]) + for r in self.store.conn.execute(query) + ] + training_rows = [ + { + "image_path": r["image_path"], + "instruction": r["prompt"], + "response": r["correction"], # human-corrected + "script": next( + (t.split(":")[1] for t in r["tags"] if t.startswith("script:")), + "unknown" + ), + } + for r in rows + ] + return Dataset.from_list(training_rows) +``` + +### UI: OCR correction view (Field Notes tab, new subtab) + +``` +[OCR Pipeline Output dir: ____] [Confidence threshold: 0.85] [Ingest] + +Table of uncertain predictions: + | Image | OCR text | Confidence | Your correction | Save | + | [img] | "Rechung 18. Ap" | 0.73 | [____________] | [✓] | + | [img] | "Beschlußprotoko" | 0.69 | [____________] | [✓] | + +[Export corrections as training dataset] [Start LoRA retrain] +``` + +### Connection to the active learning loop + +Your OCR pipeline already has: +- `abstention logic` (the "council abstains" on uncertain predictions) +- `acceptance-gated fine-tuning` +- `RAG-based post-correction` + +The workbench hook provides the missing UI layer: human-in-the-loop corrections that feed +the acceptance gate. The `FieldNoteStore.to_hf_dataset()` output plugs directly into the +pipeline's `acceptance-gated fine-tuning` step. + +--- + +## 9. MiniCPM Desk-Pet + +### What it is + +OpenBMB ships `MiniCPM-Desk-Pet`, a desktop companion app powered by MiniCPM5-1B, alongside +the model release (2026-05-19). Key features: +- Runs locally on Apple Silicon, NVIDIA GPU, or CPU +- LoRA persona switching — different personalities loaded as adapters +- Integrates with coding agents (Cursor, Claude Code, Codex) +- Tiny footprint (~2GB VRAM with Q4_K_M) + +The workbench extension lets you train LoRA personas directly and export them to the Desk-Pet +format. + +### What "persona" means here + +A LoRA persona is a small adapter (rank 8–16) trained on ~100–500 conversation examples in a +specific voice or style. Examples: +- "Botanist assistant" — answers in scientific plant terminology +- "Friendly field guide" — casual, encouraging tone for beginners +- "Historical document expert" — formal, precise, citation-aware (connects to OCR pipeline) + +Training data is small enough that synthetic gen (Extension 4) can produce it in minutes. + +### Architecture + +``` +agent/desk_pet.py + DeskPetExporter + .train_persona(name, style_desc, n_examples) → LoRA checkpoint + .export_to_deskpet(checkpoint_path) → deskpet_compatible.gguf + .list_personas() → [PersonaMeta] + .load_persona(name) → activates adapter in current session +``` + +```python +# agent/desk_pet.py +from dataclasses import dataclass +from pathlib import Path +import json, shutil + +@dataclass +class PersonaMeta: + name: str + description: str + checkpoint: str + gguf_path: str | None = None + n_examples: int = 0 + +class DeskPetExporter: + PERSONA_DIR = Path("data/personas") + + def __init__(self, base_model_id: str = "openbmb/MiniCPM5-1B"): + self.base_model_id = base_model_id + self.PERSONA_DIR.mkdir(parents=True, exist_ok=True) + + def train_persona( + self, + name: str, + style_desc: str, + n_examples: int = 200, + lora_rank: int = 8, # small rank: personas need only ~50-100 examples + ) -> str: + """ + 1. Use SyntheticGenerator to create conversation examples in the persona style. + 2. Fine-tune MiniCPM5-1B LoRA. + 3. Save checkpoint. + Returns checkpoint path. + """ + from datasets.synthetic import SyntheticGenerator + from training.lora import LoRATextTrainer + from models.minicpm_text import MiniCPMTextService + import torch + + # Generate persona training data + from transformers import AutoModelForCausalLM, AutoTokenizer + base_model = AutoModelForCausalLM.from_pretrained( + self.base_model_id, torch_dtype=torch.bfloat16, device_map="auto" + ) + base_tok = AutoTokenizer.from_pretrained(self.base_model_id) + + gen_svc = MiniCPMTextService.__new__(MiniCPMTextService) + gen_svc.model, gen_svc.tokenizer = base_model, base_tok + + synth = SyntheticGenerator(gen_svc) + ds = synth.generate( + topic=f"Conversation examples in the style of: {style_desc}. " + f"Each example: a user message and a response in that persona's voice.", + n=n_examples, + schema={"instruction": "user message", "response": "persona reply"}, + ) + + # LoRA fine-tune + trainer = LoRATextTrainer(cfg={ + "lora_rank": lora_rank, + "lora_alpha": lora_rank * 2, + "epochs": 2, + "batch_size": 8, + "grad_accum": 2, + }) + output_dir = str(self.PERSONA_DIR / name / "checkpoint") + trainer.train(base_model, base_tok, ds, run_name=f"persona_{name}") + + # Save metadata + meta = PersonaMeta( + name=name, description=style_desc, + checkpoint=output_dir, n_examples=n_examples + ) + (self.PERSONA_DIR / name / "meta.json").write_text( + json.dumps(meta.__dict__, indent=2) + ) + return output_dir + + def export_to_deskpet(self, persona_name: str) -> str: + """ + Merge LoRA into base weights, then export as GGUF for Desk-Pet. + Returns path to merged GGUF. + """ + from peft import PeftModel + from transformers import AutoModelForCausalLM, AutoTokenizer + import torch + from training.export import GGUFExporter + + meta_path = self.PERSONA_DIR / persona_name / "meta.json" + meta = PersonaMeta(**json.loads(meta_path.read_text())) + + # Merge LoRA into base + base = AutoModelForCausalLM.from_pretrained( + self.base_model_id, torch_dtype=torch.bfloat16 + ) + peft_model = PeftModel.from_pretrained(base, meta.checkpoint) + merged = peft_model.merge_and_unload() + + merged_path = str(self.PERSONA_DIR / persona_name / "merged") + merged.save_pretrained(merged_path) + AutoTokenizer.from_pretrained(self.base_model_id).save_pretrained(merged_path) + + # Export GGUF + exporter = GGUFExporter() + gguf_path = exporter.export( + model_path=merged_path, + output_dir=str(self.PERSONA_DIR / persona_name / "gguf"), + quant="Q4_K_M", + model_type="text", + )[1] # [0] = F16, [1] = quantized + + # Update metadata + meta.gguf_path = gguf_path + meta_path.write_text(json.dumps(meta.__dict__, indent=2)) + return gguf_path +``` + +### Usage flow + +``` +1. UI: "New persona" → enter name + style description +2. Synthetic gen: 200 examples of that voice → fine-tune LoRA (rank 8, ~10 min on RTX) +3. Export → Q4_K_M GGUF +4. Copy to Desk-Pet personas/ dir +5. Desk-Pet: switch persona → instant personality change +``` + +--- + +## 10. MiniCPM-o Audio Tab + +### What it is + +MiniCPM-o-4.5 (released 2026-05-17) is a true omnimodal model — it sees, listens, and speaks +simultaneously in real-time. It supports proactive interactions (proactive reminding) and +real-time conversation with both visual and audio input. + +This extension adds a new Gradio tab with a microphone + camera (or image) interface, streaming +audio output, and real-time MiniCPM-o inference. + +### Architecture + +``` +ui/audio_tab.py + OmnimodalTab + .build() → gr.Column with audio+image inputs and streaming output + +models/minicpm_omni.py + MiniCPMOmniService + .stream_chat(audio_bytes, image=None, text=None) → Generator[str] + .speak(text) → bytes (TTS for audio output) +``` + +```python +# models/minicpm_omni.py +import torch +import numpy as np +from transformers import AutoProcessor, AutoModel + +class MiniCPMOmniService: + """ + MiniCPM-o-4.5: omnimodal service. + Handles text + image + audio simultaneously. + """ + MODEL_ID = "openbmb/MiniCPM-o-4.5" + + def __init__(self, cfg: dict): + self.processor = AutoProcessor.from_pretrained( + self.MODEL_ID, trust_remote_code=True + ) + self.model = AutoModel.from_pretrained( + self.MODEL_ID, + trust_remote_code=True, + torch_dtype=torch.bfloat16, + device_map="auto", + ) + self.thinking = cfg.get("thinking_mode", False) + + def chat( + self, + text: str | None = None, + image=None, # PIL Image + audio: np.ndarray | None = None, + sample_rate: int = 16000, + ) -> str: + """ + Full omnimodal chat: pass any combination of text, image, audio. + MiniCPM-o-4.5 handles them natively. + """ + content = [] + if image is not None: + content.append({"type": "image", "image": image}) + if audio is not None: + content.append({ + "type": "audio", + "audio": audio, + "sample_rate": sample_rate, + }) + if text: + content.append({"type": "text", "text": text}) + + messages = [{"role": "user", "content": content}] + inputs = self.processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True, + chat_template_kwargs={"enable_thinking": self.thinking}, + ).to(self.model.device) + + with torch.no_grad(): + outputs = self.model.generate(**inputs, max_new_tokens=512) + return self.processor.decode( + outputs[0][inputs["input_ids"].shape[1]:], + skip_special_tokens=True + ) +``` + +```python +# ui/audio_tab.py +import gradio as gr +import numpy as np + +def build_audio_tab(model_registry): + with gr.Column(): + gr.Markdown("### 🎙️ Omnimodal — MiniCPM-o-4.5") + gr.Markdown("Speak, show an image, or type — all at once.") + + with gr.Row(): + audio_in = gr.Audio( + sources=["microphone"], + type="numpy", + label="Microphone input", + streaming=True, + ) + image_in = gr.Image( + type="pil", + label="Optional image", + ) + + text_in = gr.Textbox(label="Optional text", placeholder="Or type here...") + submit = gr.Button("Send") + output = gr.Textbox(label="Response", lines=8) + audio_out = gr.Audio(label="Spoken response (TTS)", autoplay=True) + + def respond(audio_data, image, text): + svc = model_registry.get("minicpm_o45") + audio_arr = None + if audio_data is not None: + sr, arr = audio_data + audio_arr = arr.astype(np.float32) / 32768.0 + response = svc.chat(text=text, image=image, audio=audio_arr) + # Optional: TTS for spoken output + # spoken = tts(response) + return response, None # None = no audio out yet + + submit.click(respond, [audio_in, image_in, text_in], [output, audio_out]) +``` + +### Real-time streaming version (advanced) + +For true real-time conversation (proactive reminding, interrupt detection): + +```python +# streaming audio inference using gradio's streaming audio + SSE +demo = gr.Interface( + fn=stream_omni_response, + inputs=[ + gr.Audio(streaming=True, sources=["microphone"]), + gr.Image(type="pil"), + ], + outputs=gr.Textbox(), + live=True, +) +``` + +### TTS for audio output + +MiniCPM-o-4.5 has its own audio generation capability — check the model card for the +`generate_audio` method. Fallback: use `kokoro-82M` (Apache 2.0, 82M, runs locally) for TTS. + +```python +# Kokoro TTS fallback (lightweight, local) +from kokoro import KPipeline +tts_pipe = KPipeline(lang_code="en-us") +audio, sr = tts_pipe(response_text, voice="af_heart") +``` + +--- + +## 11. Cross-Extension Wiring + +Most extensions are independent, but several combinations unlock powerful compound workflows: + +### OCR → VINDEX + +Low-confidence OCR outputs → Field Notes → human corrections → LoRA retrain. But additionally: +if the OCR model consistently misreads a specific character class, use VINDEX's `logit_lens` to +identify which layer is responsible, then `calibrated_edit` to target that slot directly — a +faster fix than a full retraining cycle. + +### Synthetic Gen → Reward Model → DPO + +``` +SyntheticGenerator.generate(topic, n=1000) + → RewardEvaluator.create_dpo_pairs(prompts, generator, n=4) + → DPO training via TRL DPOTrainer + → Trackio logs win rate + → VINDEX verify alignment not broken +``` + +### Paper Agent → Desk-Pet Persona + +``` +PaperAgent reads: "Persona-based dialogue systems for domain experts" + → Implements: persona training data format + → DeskPetExporter.train_persona("expert_botanist", "...") + → Export GGUF → load in Desk-Pet +``` + +### HF Spaces + vLLM + Trackio + +``` +SpacesDeployer.deploy(hardware="a10") # production GPU + → app.py runs VLLMRunner on A10G + → All requests logged via Trackio with space_id=deployed_space + → Dashboard visible to collaborators at trackio Space URL +``` + +### Full active-learning loop (all extensions combined) + +``` +MiniCPM-o Audio Tab: user speaks + shows image of plant + ↓ +OmnimodalService: identify species (low confidence) + ↓ +OCRPipelineLoader: auto-create Field Note (uncertain prediction) + ↓ +Human: corrects species name in UI + ↓ +SyntheticGenerator: augment with 50 similar examples + ↓ +RewardEvaluator: filter synthetic examples + ↓ +LoRATextTrainer + TRL + Trackio: fine-tune + ↓ +VINDEX: verify the target species slot was corrected + ↓ +DeskPetExporter: export updated persona + ↓ +SpacesDeployer: push updated app to HF Spaces +``` + +--- + +*Extensions spec v1.0 — Christof Kaller / ki-fusion-labs.de — 2026-06-05* \ No newline at end of file diff --git a/HF_PRD_v1.md b/HF_PRD_v1.md new file mode 100644 index 0000000000000000000000000000000000000000..340ff640b957ec96d5c230f83104cd1a8bbdba90 --- /dev/null +++ b/HF_PRD_v1.md @@ -0,0 +1,1361 @@ +# OpenBMB Local AI Workbench — Technical PRD v2.0 +> Revised: 2026-06-05 | Corrections from online research embedded inline + +--- + +## Table of Contents + +1. [Purpose & Design Philosophy](#1-purpose--design-philosophy) +2. [Design Principles (Template Architecture)](#2-design-principles-template-architecture) +3. [System Architecture](#3-system-architecture) +4. [Model Registry](#4-model-registry) +5. [Inference Stack — Five Modes](#5-inference-stack--five-modes) +6. [Tracking: Trackio (Corrected API)](#6-tracking-trackio-corrected-api) +7. [MCP Layer (Three Integration Paths)](#7-mcp-layer-three-integration-paths) +8. [Training Pipeline](#8-training-pipeline) +9. [Export & Quantization](#9-export--quantization) +10. [Agent Mode (ml-intern Inspired)](#10-agent-mode-ml-intern-inspired) +11. [UI — Tab Specification](#11-ui--tab-specification) +12. [Field Notes & Correction Loop](#12-field-notes--correction-loop) +13. [Directory Structure](#13-directory-structure) +14. [Configuration Schema](#14-configuration-schema) +15. [Dependencies](#15-dependencies) +16. [Hackathon Demo Flow](#16-hackathon-demo-flow) +17. [Corrections from PRD v1](#17-corrections-from-prd-v1) +18. [Roadmap & Extension Points](#18-roadmap--extension-points) + +--- + +## 1. Purpose & Design Philosophy + +**One-line:** A modular AI experimentation platform for the OpenBMB model family that covers +dataset ingestion → LoRA training → evaluation → GGUF export → llama.cpp/SGLang deployment → +multimodal inference → MCP tool exposure → trace sharing — in a single Gradio app. + +**Broader framing:** The platform is deliberately designed as a *template*. The `config/models.yaml` +and `config/training.yaml` files drive almost all model-specific behaviour. Replacing the OpenBMB +section with any other model family (Qwen, Phi, Gemma …) should require only config changes and +a matching service class, never core rewrites. + +**Why OpenBMB?** +- MiniCPM5-1B (2026-05-19): SOTA 1B on-device, 128K context, LlamaForCausalLM — no custom kernels +- MiniCPM4.1-8B (2025-09): sparse InfLLM-v2 attention, ~7× long-context speedup vs Qwen3-8B +- MiniCPM-V-4.6 (2026-05-11): 1.3B VLM, SigLIP2-400M + Qwen3.5-0.8B, 262K ctx, mixed 4×/16× token compression +- MiniCPM-V-4.6-Thinking: same architecture + long-CoT for multimodal reasoning +- MiniCPM-o 4.5 (2026-05-17): omnimodal (vision + audio), real-time conversation +- Apache 2.0 across the board; vLLM / SGLang / llama.cpp / Ollama native support + +--- + +## 2. Design Principles (Template Architecture) + +| Principle | Implementation | +|-----------|----------------| +| Config-driven | `models.yaml` + `training.yaml` define all models; no model names hardcoded in logic | +| Event-driven | `core/events.py` — typed event bus; all cross-module comms via events | +| Registry pattern | `core/registry.py` — all services register by name; swap without restart | +| Separation of concerns | `models/` = loading; `training/` = train/eval; `ui/` = Gradio; `mcp/` = protocol | +| Plugin tools | Any Python function decorated `@mcp.tool()` is auto-exposed; no wiring needed | +| Async-first | FastAPI backend, Gradio queuing, non-blocking inference | +| Observability | Every significant action fires a Trackio log + event | +| Template portability | Replace `config/models.yaml` → different model family; keep all else | + +--- + +## 3. System Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ OpenBMB Local AI Workbench │ +├───────────────────────────┬─────────────────────────────────────────────────┤ +│ Gradio UI Layer │ Backend Service Layer │ +│ ┌─────────────────────┐ │ ┌─────────────────────────────────────────┐ │ +│ │ gr.Blocks / Tabs │ │ │ gradio.Server (FastAPI-based) │ │ +│ │ chat_tab.py │ │ │ @app.api() — Gradio-queued endpoints │ │ +│ │ vision_tab.py │ │ │ @app.mcp.tool() — MCP registration │ │ +│ │ train_tab.py │ │ │ @app.get("/") — static/custom frontend │ │ +│ │ dataset_tab.py │ │ └─────────────────────────────────────────┘ │ +│ │ export_tab.py │ │ │ +│ │ traces_tab.py │ │ ┌─────────────────────────────────────────┐ │ +│ │ agent_tab.py │ │ │ FastMCP server (standalone / bridged) │ │ +│ └─────────────────────┘ │ │ mcp = FastMCP("OpenBMBWorkbench") │ │ +│ │ │ Registered tools (see §7) │ │ +│ SSE MCP endpoint: │ └─────────────────────────────────────────┘ │ +│ /gradio_api/mcp/sse │ │ +└───────────────────────────┴─────────────────────────────────────────────────┘ + │ │ + ▼ ▼ +┌─────────────────────┐ ┌────────────────────────┐ +│ Model Services │ │ Tracking & Storage │ +│ minicpm_text.py │ │ trackio (local/Space) │ +│ minicpm_vision.py │ │ field_notes/ (SQLite) │ +│ llama_cpp_runner.py│ │ exports/ (GGUF files) │ +│ sglang_runner.py │ │ data/ (HF datasets) │ +│ ollama_runner.py │ └────────────────────────┘ +└─────────────────────┘ + │ + ▼ +┌───────────────────────────────────────────────────────────┐ +│ Inference Backends │ +│ Transformers │ llama.cpp │ SGLang │ vLLM │ Ollama│ +└───────────────────────────────────────────────────────────┘ +``` + +### 3.1 Event Bus (core/events.py) + +```python +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable +import asyncio + +class EventType(str, Enum): + DATASET_LOADED = "dataset_loaded" + TRAINING_STARTED = "training_started" + TRAINING_STEP = "training_step" + TRAINING_FINISHED = "training_finished" + EVAL_FINISHED = "eval_finished" + INFERENCE_REQUEST = "inference_request" + INFERENCE_RESPONSE = "inference_response" + TOOL_CALL = "tool_call" + MODEL_LOADED = "model_loaded" + MODEL_SWITCHED = "model_switched" + EXPORT_STARTED = "export_started" + EXPORT_FINISHED = "export_finished" + FIELD_NOTE_SAVED = "field_note_saved" + AGENT_STEP = "agent_step" + +@dataclass +class Event: + type: EventType + payload: dict[str, Any] = field(default_factory=dict) + run_id: str = "" + +class EventBus: + _handlers: dict[EventType, list[Callable]] = {} + + def on(self, event_type: EventType): + """Decorator to register an event handler.""" + def decorator(fn: Callable): + self._handlers.setdefault(event_type, []).append(fn) + return fn + return decorator + + async def emit(self, event: Event): + for handler in self._handlers.get(event.type, []): + await handler(event) + +bus = EventBus() +``` + +### 3.2 Service Registry (core/registry.py) + +```python +from typing import TypeVar, Generic, Type + +T = TypeVar("T") + +class Registry(Generic[T]): + def __init__(self): + self._services: dict[str, T] = {} + + def register(self, name: str, service: T): + self._services[name] = service + + def get(self, name: str) -> T: + if name not in self._services: + raise KeyError(f"Service '{name}' not registered. Available: {list(self._services)}") + return self._services[name] + + def list(self) -> list[str]: + return list(self._services.keys()) + +# Singleton registries +model_registry = Registry() # model services +dataset_registry = Registry() # dataset loaders +tool_registry = Registry() # MCP tools +``` + +--- + +## 4. Model Registry + +### 4.1 Text Models + +| Config ID | HF ID | Architecture | Context | Notes | +|-----------|-------|-------------|---------|-------| +| `minicpm5_1b` | `openbmb/MiniCPM5-1B` | LlamaForCausalLM | 128K | Standard; no custom kernels; XML tool calls; SGLang `minicpm5` parser | +| `minicpm5_1b_thinking` | `openbmb/MiniCPM5-1B-Thinking` | LlamaForCausalLM | 128K | CoT mode; `chat_template_kwargs={"enable_thinking": True}` | +| `minicpm41_8b` | `openbmb/MiniCPM4.1-8B` | Sparse InfLLM-v2 | 128K (YaRN) | `--trust-remote-code` required; 7× faster long-ctx vs Qwen3-8B | + +### 4.2 Vision Models + +| Config ID | HF ID | Backbone | Context | Notes | +|-----------|-------|---------|---------|-------| +| `minicpm_v46` | `openbmb/MiniCPM-V-4.6` | SigLIP2-400M + Qwen3.5-0.8B | 262K | `transformers>=5.7.0` + `torchcodec`; mixed 4×/16× compression | +| `minicpm_v46_thinking` | `openbmb/MiniCPM-V-4.6-Thinking` | same + CoT head | 262K | `enable_thinking=True` in chat_template_kwargs | +| `minicpm_o45` | `openbmb/MiniCPM-o-4.5` | Omnimodal | - | Vision + Audio; real-time conversation; see §4.4 | + +### 4.3 Model Architecture Notes (critical for correct code) + +**MiniCPM5-1B** — standard Llama, loads with: +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +model = AutoModelForCausalLM.from_pretrained("openbmb/MiniCPM5-1B", trust_remote_code=True) +``` + +**MiniCPM-V-4.6** — vision-language, loads with: +```python +# ⚠️ CORRECTION from v1: NOT AutoModelForCausalLM — use AutoModelForImageTextToText +from transformers import AutoProcessor, AutoModelForImageTextToText + +processor = AutoProcessor.from_pretrained("openbmb/MiniCPM-V-4.6", trust_remote_code=True) +model = AutoModelForImageTextToText.from_pretrained( + "openbmb/MiniCPM-V-4.6", + trust_remote_code=True, + torch_dtype=torch.bfloat16, + device_map="auto" +) +``` + +Chat format (vision): +```python +messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": pil_image}, # PIL Image + {"type": "text", "text": prompt} + ] + } +] +inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True, + chat_template_kwargs={"enable_thinking": False} # True for Thinking variant +).to(model.device) +``` + +**MiniCPM4.1-8B** — requires `trust_remote_code=True` for sparse attention kernel. + +### 4.4 models.yaml Schema + +```yaml +# config/models.yaml +# ─── Template: add any HF model here ────────────────────────────────────── +models: + minicpm5_1b: + hf_id: openbmb/MiniCPM5-1B + type: text # text | vision | omnimodal + architecture: llama # llama | minicpm41 | minicpm_v + context_length: 131072 + dtype: bfloat16 + trust_remote_code: false # LlamaForCausalLM — no fork + thinking_mode: false # set true for Thinking checkpoints + tool_call_parser: minicpm5 # for SGLang serving + gguf: + available: true + repo: openbmb/MiniCPM5-1B-GGUF + default_quant: Q4_K_M + + minicpm41_8b: + hf_id: openbmb/MiniCPM4.1-8B + type: text + architecture: minicpm41 # sparse InfLLM-v2 + context_length: 131072 + dtype: bfloat16 + trust_remote_code: true + thinking_mode: false + gguf: + available: true + repo: openbmb/MiniCPM4.1-8B-GGUF + default_quant: Q4_K_M + + minicpm_v46: + hf_id: openbmb/MiniCPM-V-4.6 + type: vision + architecture: minicpm_v + context_length: 262144 + dtype: bfloat16 + trust_remote_code: true + thinking_mode: false + vision_encoder: siglip2_400m + token_compression: "4x_16x" # mixed compression + gguf: + available: true + repo: openbmb/MiniCPM-V-4.6-gguf # official repo — no surgery script needed + main_file: MiniCPM-V-4.6-Q4_K_M.gguf + mmproj: mmproj-MiniCPM-V-4.6-F16.gguf + default_quant: Q4_K_M + + minicpm_v46_thinking: + hf_id: openbmb/MiniCPM-V-4.6-Thinking + type: vision + architecture: minicpm_v + context_length: 262144 + dtype: bfloat16 + trust_remote_code: true + thinking_mode: true + gguf: + available: true + repo: openbmb/MiniCPM-V-4.6-Thinking-gguf + default_quant: Q4_K_M + + minicpm_o45: + hf_id: openbmb/MiniCPM-o-4.5 + type: omnimodal + architecture: minicpm_o + context_length: 131072 + dtype: bfloat16 + trust_remote_code: true + thinking_mode: false + note: "Vision + Audio; real-time conversation mode" +``` + +--- + +## 5. Inference Stack — Five Modes + +### Mode A: Transformers (train + eval + fine-tuning) + +Used for: LoRA training, evaluation, PEFT +Load text: `AutoModelForCausalLM` +Load vision: `AutoModelForImageTextToText` ← corrected from v1 + +```python +# models/minicpm_text.py +class MiniCPMTextService: + def __init__(self, model_id: str, cfg: dict): + self.model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", + trust_remote_code=cfg.get("trust_remote_code", False) + ) + self.tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) +``` + +```python +# models/minicpm_vision.py ← new dedicated service +class MiniCPMVisionService: + def __init__(self, model_id: str, cfg: dict): + self.processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True) + self.model = AutoModelForImageTextToText.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", + trust_remote_code=cfg.get("trust_remote_code", True) + ) + self.thinking = cfg.get("thinking_mode", False) + + def chat(self, messages: list[dict], max_new_tokens: int = 512) -> str: + inputs = self.processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True, + chat_template_kwargs={"enable_thinking": self.thinking} + ).to(self.model.device) + with torch.no_grad(): + outputs = self.model.generate(**inputs, max_new_tokens=max_new_tokens) + response = self.processor.decode( + outputs[0][inputs["input_ids"].shape[1]:], + skip_special_tokens=True + ) + return response +``` + +### Mode B: llama.cpp (local deployment, CPU/GPU offload) + +> ⚠️ CORRECTION from v1: For MiniCPM-V 4.6, official GGUFs are already published. +> Download from `openbmb/MiniCPM-V-4.6-gguf` — no need to run `convert_hf_to_gguf.py` yourself. +> The `minicpmv-surgery.py` script was for the older MiniCPM-o-2.6; **do not use it for V4.6**. + +**Text models** → `llama-server` (standard OpenAI-compatible endpoint) +**Vision models** → `llama-server --mmproj ` (multimodal server) + +```python +# models/llama_cpp_runner.py +import subprocess, requests, time + +class LlamaCppDeployment: + def __init__(self, cfg: dict): + self.model_path = cfg["model_path"] + self.mmproj_path = cfg.get("mmproj_path") # None for text models + self.port = cfg.get("port", 8080) + self._proc = None + + def start(self): + cmd = [ + "llama-server", + "--model", self.model_path, + "--port", str(self.port), + "--ctx-size", "8192", + "--n-gpu-layers", "-1", # full GPU offload if available + ] + if self.mmproj_path: + cmd += ["--mmproj", self.mmproj_path] # enables vision + self._proc = subprocess.Popen(cmd) + self._wait_ready() + + def _wait_ready(self, timeout: int = 30): + for _ in range(timeout): + try: + r = requests.get(f"http://127.0.0.1:{self.port}/health") + if r.status_code == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("llama-server did not start in time") + + def chat(self, messages: list[dict]) -> str: + r = requests.post( + f"http://127.0.0.1:{self.port}/v1/chat/completions", + json={"messages": messages, "max_tokens": 512} + ) + return r.json()["choices"][0]["message"]["content"] + + def stop(self): + if self._proc: + self._proc.terminate() +``` + +**GGUF Export** (only needed if building custom quants): +```bash +# For MiniCPM-V 4.6 — convert_hf_to_gguf.py (NOT legacy surgery scripts) +python convert_hf_to_gguf.py openbmb/MiniCPM-V-4.6 --outtype f16 +# Output: MiniCPM-V-4.6-F16.gguf + mmproj-MiniCPM-V-4.6-F16.gguf + +# Quantize +./llama-quantize MiniCPM-V-4.6-F16.gguf MiniCPM-V-4.6-Q4_K_M.gguf Q4_K_M +``` + +Supported quants: `Q4_K_M`, `Q5_K_M`, `Q8_0`, `Q2_K`, `F16` + +### Mode C: SGLang (tool-use, function calling, recommended for MiniCPM5) + +MiniCPM5-1B emits XML-style tool calls. SGLang has a native `minicpm5` parser: + +```bash +python -m sglang.launch_server \ + --model-path openbmb/MiniCPM5-1B \ + --port 30000 \ + --tool-call-parser minicpm5 # or: --tool-call-parser auto +``` + +For MiniCPM-V-4.6 vision: +```bash +python -m sglang.launch_server \ + --model-path openbmb/MiniCPM-V-4.6-Thinking \ + --port 30000 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --default-chat-template-kwargs '{"enable_thinking": true}' +``` + +### Mode D: vLLM (batch serving, production) + +```bash +vllm serve openbmb/MiniCPM4.1-8B --trust-remote-code +vllm serve openbmb/MiniCPM-V-4.6 --trust-remote-code +``` + +### Mode E: Ollama (zero-config, consumer hardware) + +```bash +ollama run openbmb/minicpm-v4.6 # vision + text +ollama run openbmb/minicpm5-1b # text only +``` + +### Mode Selection Matrix + +| Use case | Recommended mode | +|----------|-----------------| +| LoRA training | A (Transformers) | +| Evaluation | A (Transformers) | +| Local chat (fast) | B (llama.cpp) | +| Tool/function calling | C (SGLang) | +| Production batch | D (vLLM) | +| Consumer demo | E (Ollama) | +| Vision inference local | B (llama-server --mmproj) | +| Vision fine-tuning | A (SWIFT / LLaMA-Factory) | + +--- + +## 6. Tracking: Trackio (Corrected API) + +> ⚠️ CRITICAL CORRECTION from v1: `trackio.trace()` **does not exist**. +> There is no context-manager trace API in Trackio. The v1 PRD example was wrong. +> Correct API: `init()` / `log()` / `finish()` / `alert()` / `Table()` / `Markdown()` + +Trackio is a wandb drop-in replacement, local-first, free, Gradio dashboard, HF Spaces sync. + +```python +# tracking/trackio_service.py +import trackio +import time +from core.events import bus, EventType, Event + +class TrackioService: + def __init__(self, project: str, space_id: str | None = None): + self.project = project + self.space_id = space_id + self._active = False + self._register_handlers() + + def _register_handlers(self): + @bus.on(EventType.TRAINING_STARTED) + async def on_training_started(event: Event): + trackio.init( + project=self.project, + space_id=self.space_id, + run_name=event.payload.get("run_name", "training"), + ) + self._active = True + + @bus.on(EventType.TRAINING_STEP) + async def on_step(event: Event): + if self._active: + trackio.log(event.payload) + + @bus.on(EventType.TRAINING_FINISHED) + async def on_done(event: Event): + if self._active: + trackio.log({"status": "finished", **event.payload}) + trackio.finish() + self._active = False + + @bus.on(EventType.EVAL_FINISHED) + async def on_eval(event: Event): + trackio.init( + project=self.project, + space_id=self.space_id, + run_name=event.payload.get("eval_run_name", "eval"), + ) + trackio.log(event.payload) + trackio.finish() + + @bus.on(EventType.INFERENCE_RESPONSE) + async def on_inference(event: Event): + # Log inference latency and token count + trackio.init( + project=self.project, + space_id=self.space_id, + run_name="inference", + ) + trackio.log({ + "model": event.payload.get("model"), + "latency_ms": event.payload.get("latency_ms"), + "tokens_out": event.payload.get("tokens_out"), + "mode": event.payload.get("mode", "text"), + }) + trackio.finish() + + @bus.on(EventType.TOOL_CALL) + async def on_tool(event: Event): + trackio.alert( + title=f"Tool call: {event.payload.get('tool_name')}", + level=trackio.AlertLevel.INFO, + ) +``` + +**TRL LoRA integration** (recommended for training): +```python +from trl import SFTConfig +training_args = SFTConfig( + output_dir="./checkpoints", + report_to="trackio", # ← one-liner Trackio integration via TRL + run_name="lora_minicpm5_1b", +) +``` + +**SQL query via CLI** (LLM-friendly): +```bash +trackio query project --project workbench --sql "SELECT run_name, loss FROM metrics ORDER BY loss LIMIT 5" +``` + +--- + +## 7. MCP Layer (Three Integration Paths) + +### Path 1: Gradio Blocks + `launch(mcp_server=True)` ← simplest + +```python +# app.py (simple path) +import gradio as gr +from mcp.server.fastmcp import FastMCP + +demo = gr.Blocks() +# ... tabs ... +demo.launch( + mcp_server=True, # exposes /gradio_api/mcp/sse + server_port=7860, +) +``` + +Every `gr.Interface` function and `gr.api()` endpoint becomes an MCP tool automatically. +Docstrings become tool descriptions. Type hints become parameter schemas. + +### Path 2: `gradio.Server` (custom frontend + MCP) + +> New API published 2026-04-01. Extends FastAPI. Better for custom dashboards. + +```python +# app.py (Server path) +from gradio import Server +from fastapi.responses import HTMLResponse + +app = Server() + +@app.api(name="run_inference") # Gradio-queued + MCP-compatible +def run_inference(prompt: str, model_id: str) -> str: + """Run text inference on a MiniCPM text model.""" + svc = model_registry.get(model_id) + return svc.generate(prompt) + +@app.mcp.tool() # explicit MCP tool registration +async def export_gguf(model_id: str, quant: str = "Q4_K_M") -> str: + """Export a Transformers model to GGUF format with specified quantization.""" + return await export_service.run(model_id, quant) + +@app.get("/", response_class=HTMLResponse) # custom frontend +async def homepage(): + return open("ui/index.html").read() + +app.launch(show_error=True) +``` + +### Path 3: FastMCP standalone + Gradio client bridge + +Use when you need stateful MCP tools, or when running multiple Gradio apps and want to save memory: + +```python +# mcp/server.py +from mcp.server.fastmcp import FastMCP +from gradio_client import Client + +mcp = FastMCP("OpenBMBWorkbench") +_clients: dict[str, Client] = {} + +def _gradio_client(space_id: str) -> Client: + if space_id not in _clients: + _clients[space_id] = Client(space_id) + return _clients[space_id] + +@mcp.tool() +async def calculate(expression: str) -> str: + """Evaluate a mathematical expression.""" + import math + return str(eval(expression, {"__builtins__": {}}, vars(math))) + +@mcp.tool() +async def dataset_stats(dataset_name: str, split: str = "train") -> dict: + """Return basic statistics for a HuggingFace dataset.""" + from datasets import load_dataset + ds = load_dataset(dataset_name, split=split) + return { + "rows": len(ds), + "columns": ds.column_names, + "features": str(ds.features), + } + +@mcp.tool() +async def search_dataset(query: str, max_results: int = 5) -> list[dict]: + """Search HuggingFace Hub for datasets matching a query.""" + from huggingface_hub import list_datasets + results = list(list_datasets(search=query, limit=max_results)) + return [{"id": d.id, "downloads": d.downloads, "tags": d.tags} for d in results] + +@mcp.tool() +async def start_training( + model_id: str, + dataset_name: str, + lora_rank: int = 16, + epochs: int = 3, +) -> dict: + """Start a LoRA fine-tuning run on the specified model and dataset.""" + # Dispatch to training service via event bus + await bus.emit(Event( + type=EventType.TRAINING_STARTED, + payload={"model_id": model_id, "dataset": dataset_name, + "lora_rank": lora_rank, "epochs": epochs} + )) + return {"status": "started", "model_id": model_id} + +@mcp.tool() +async def evaluate_model(model_id: str, dataset_name: str) -> dict: + """Evaluate a model on a dataset and return metrics.""" + svc = model_registry.get(model_id) + return await eval_service.run(svc, dataset_name) + +@mcp.tool() +async def export_gguf(model_id: str, quant: str = "Q4_K_M") -> str: + """Convert a fine-tuned model to GGUF format.""" + return await export_service.run(model_id, quant) + +@mcp.tool() +async def vision_chat(image_path: str, prompt: str, model_id: str = "minicpm_v46") -> str: + """Run multimodal inference on an image with a prompt.""" + svc: MiniCPMVisionService = model_registry.get(model_id) + from PIL import Image + img = Image.open(image_path).convert("RGB") + messages = [{"role": "user", "content": [ + {"type": "image", "image": img}, + {"type": "text", "text": prompt}, + ]}] + return svc.chat(messages) + +if __name__ == "__main__": + mcp.run(transport="sse", port=8081) +``` + +MCP SSE endpoint: `http://localhost:8081/sse` +Gradio MCP SSE: `http://localhost:7860/gradio_api/mcp/sse` + +--- + +## 8. Training Pipeline + +### 8.1 LoRA — Text Models (PEFT) + +```python +# training/lora.py +from peft import LoraConfig, get_peft_model, TaskType +from trl import SFTTrainer, SFTConfig + +class LoRATextTrainer: + DEFAULT_TARGET_MODULES = ["q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj"] + + def __init__(self, cfg: dict): + self.cfg = cfg + + def train(self, model, tokenizer, dataset, run_name: str): + lora_cfg = LoraConfig( + r=self.cfg.get("lora_rank", 16), + lora_alpha=self.cfg.get("lora_alpha", 32), + target_modules=self.cfg.get("target_modules", self.DEFAULT_TARGET_MODULES), + lora_dropout=self.cfg.get("lora_dropout", 0.05), + bias="none", + task_type=TaskType.CAUSAL_LM, + ) + model = get_peft_model(model, lora_cfg) + model.print_trainable_parameters() + + training_args = SFTConfig( + output_dir=f"./checkpoints/{run_name}", + num_train_epochs=self.cfg.get("epochs", 3), + per_device_train_batch_size=self.cfg.get("batch_size", 4), + gradient_accumulation_steps=self.cfg.get("grad_accum", 4), + learning_rate=self.cfg.get("lr", 2e-4), + fp16=False, + bf16=True, + report_to="trackio", # ← Trackio via TRL + run_name=run_name, + save_steps=100, + logging_steps=10, + ) + trainer = SFTTrainer( + model=model, + args=training_args, + train_dataset=dataset, + tokenizer=tokenizer, + ) + trainer.train() + return model +``` + +### 8.2 LoRA — Vision Models (SWIFT / LLaMA-Factory recommended) + +> OpenBMB recommends SWIFT or LLaMA-Factory for MiniCPM-V fine-tuning. +> Vision tower is frozen; only language backbone layers are adapted. + +```python +# training/lora_vision.py +class LoRAVisionConfig: + """ + For MiniCPM-V 4.6 LoRA: + - Freeze vision encoder (SigLIP2-400M) + projector + - Apply LoRA only to Qwen3.5-0.8B language layers + - Use SWIFT CLI or LLaMA-Factory for full pipeline + """ + LANGUAGE_TARGET_MODULES = ["q_proj", "k_proj", "v_proj", "o_proj"] + FROZEN_PREFIXES = ["vpm", "resampler", "vision_model"] # freeze vision + + @staticmethod + def freeze_vision(model): + for name, param in model.named_parameters(): + if any(name.startswith(pfx) for pfx in LoRAVisionConfig.FROZEN_PREFIXES): + param.requires_grad = False + return model +``` + +**SWIFT CLI approach** (recommended for MiniCPM-V 4.6): +```bash +swift sft \ + --model openbmb/MiniCPM-V-4.6 \ + --dataset \ + --lora_rank 16 \ + --freeze_vit true \ + --output_dir ./checkpoints/minicpm_v46_lora +``` + +### 8.3 HF Jobs Offload (ml-intern pattern) + +For long training runs, offload to HF Jobs (free `cpu-basic`, paid `gpu-a100`): + +```python +# training/hf_jobs.py +from huggingface_hub import HfApi + +class HFJobsTrainer: + def submit(self, script_path: str, hardware: str = "gpu-a100") -> str: + api = HfApi() + job = api.create_job( + repo_id=f"{api.whoami()['name']}/ml-workbench-jobs", + script=script_path, + hardware=hardware, + ) + return job.job_id +``` + +### 8.4 Evaluation + +```python +# training/evaluation.py +class Evaluator: + def run(self, model, tokenizer, dataset, metrics=("perplexity", "accuracy")) -> dict: + results = {} + if "perplexity" in metrics: + results["perplexity"] = self._perplexity(model, tokenizer, dataset) + if "accuracy" in metrics: + results["accuracy"] = self._accuracy(model, tokenizer, dataset) + return results +``` + +--- + +## 9. Export & Quantization + +```python +# training/export.py +import subprocess +from pathlib import Path + +class GGUFExporter: + SUPPORTED_QUANTS = ["F16", "Q4_K_M", "Q5_K_M", "Q8_0", "Q2_K"] + + def export( + self, + model_path: str, + output_dir: str, + quant: str = "Q4_K_M", + model_type: str = "text", # "text" | "vision" + ) -> list[str]: + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Step 1: Convert HF → GGUF F16 + # For MiniCPM-V 4.6: convert_hf_to_gguf.py produces main gguf + mmproj + # For text models: same script, no mmproj + f16_path = output_dir / "model-f16.gguf" + subprocess.run([ + "python", "convert_hf_to_gguf.py", + model_path, + "--outtype", "f16", + "--outfile", str(f16_path), + ], check=True) + + outputs = [str(f16_path)] + + # Step 2: Quantize + if quant != "F16": + quant_path = output_dir / f"model-{quant}.gguf" + subprocess.run([ + "llama-quantize", + str(f16_path), + str(quant_path), + quant, + ], check=True) + outputs.append(str(quant_path)) + + return outputs +``` + +> Note: For MiniCPM-V 4.6, official GGUF files are already published at +> `openbmb/MiniCPM-V-4.6-gguf`. Download them directly unless you need custom quants. + +--- + +## 10. Agent Mode (ml-intern Inspired) + +The `agent/` module implements a lightweight research-plan-implement loop inspired by HF's +ml-intern (April 2026). This gives the workbench autonomous ML experimentation capabilities. + +``` +ml-intern architecture (reference): + Research → browse arXiv/HF Papers, read methodology, traverse citations + Plan → break task down, verify resources (models, datasets, hardware) + Implement → execute scripts on HF Jobs / local GPU + Trace → auto-upload sessions as JSONL to HF Dataset +``` + +```python +# agent/loop.py +from smolagents import CodeAgent, HfApiModel +from huggingface_hub import HfApi + +class WorkbenchAgent: + """ + Lightweight agent loop for autonomous experimentation. + Uses smolagents + HF ecosystem + registered MCP tools. + """ + def __init__(self, model_id: str = "claude-sonnet-4-6"): + self.model = HfApiModel(model_id) + self.api = HfApi() + self._session_log = [] + + def run(self, prompt: str, max_steps: int = 20) -> str: + agent = CodeAgent( + tools=self._get_tools(), + model=self.model, + max_steps=max_steps, + ) + result = agent.run(prompt) + self._save_session(prompt, result) + return result + + def _save_session(self, prompt: str, result: str): + import json, datetime + session = { + "timestamp": datetime.datetime.utcnow().isoformat(), + "prompt": prompt, + "result": result, + "log": self._session_log, + } + # Auto-upload to private HF dataset (ml-intern pattern) + # Dataset: {username}/workbench-sessions + pass + + def _get_tools(self): + # Expose MCP tools as smolagents tools + from mcp.server.fastmcp import FastMCP + # ... + return [] +``` + +--- + +## 11. UI — Tab Specification + +Using `gr.Blocks` with tabs. Optionally swap to `gradio.Server` for custom frontend. + +```python +# app.py +import gradio as gr +from ui.chat_tab import build_chat_tab +from ui.vision_tab import build_vision_tab +from ui.train_tab import build_train_tab +from ui.dataset_tab import build_dataset_tab +from ui.export_tab import build_export_tab +from ui.traces_tab import build_traces_tab +from ui.agent_tab import build_agent_tab +from ui.notes_tab import build_notes_tab + +with gr.Blocks(title="OpenBMB Workbench", theme=gr.themes.Ocean()) as demo: + gr.Markdown("# 🔬 OpenBMB Local AI Workbench") + + with gr.Tabs(): + with gr.Tab("💬 Chat"): build_chat_tab() + with gr.Tab("👁️ Vision"): build_vision_tab() + with gr.Tab("📊 Dataset"): build_dataset_tab() + with gr.Tab("🏋️ Train"): build_train_tab() + with gr.Tab("📦 Export"): build_export_tab() + with gr.Tab("📈 Traces"): build_traces_tab() + with gr.Tab("🤖 Agent"): build_agent_tab() + with gr.Tab("📝 Field Notes"):build_notes_tab() + +demo.launch( + mcp_server=True, # SSE at /gradio_api/mcp/sse + server_port=7860, + share=False, +) +``` + +### Tab Summary + +| Tab | Purpose | Key components | +|-----|---------|----------------| +| Chat | Text inference on all text models | Model selector, streaming output, system prompt | +| Vision | Image + text inference on V-4.6/Thinking | Image upload, thinking toggle, bounding box overlay | +| Dataset | HF Hub browser + local CSV/JSON loader | Search bar, split preview, schema inspector | +| Train | LoRA training launcher | Config form, live loss chart (Trackio), checkpoint browser | +| Export | GGUF export + quantization | Model selector, quant dropdown, file download | +| Traces | Trackio run browser + comparison | Run table, metric plots, SQL query box | +| Agent | ml-intern style agent loop | Prompt, step log, paper browser, HF Jobs monitor | +| Field Notes | Correction capture → retrain trigger | Image+prompt+response+correction form, JSONL export | + +### Vision Tab — key detail + +```python +# ui/vision_tab.py +def build_vision_tab(): + with gr.Column(): + model_dd = gr.Dropdown( + choices=["minicpm_v46", "minicpm_v46_thinking", "minicpm_o45"], + value="minicpm_v46", + label="Vision Model" + ) + thinking_cb = gr.Checkbox(label="Enable Thinking Mode", value=False) + image_in = gr.Image(type="pil", label="Upload Image") + video_in = gr.Video(label="Upload Video (MiniCPM-V-4.6 supports video)") + prompt_in = gr.Textbox(label="Prompt") + submit_btn = gr.Button("Run") + output_txt = gr.Textbox(label="Response", lines=10) + + def infer(model_id, thinking, image, video, prompt): + svc = model_registry.get(model_id) + content = [] + if image: + content.append({"type": "image", "image": image}) + if video: + content.append({"type": "video", "video": video}) + content.append({"type": "text", "text": prompt}) + msgs = [{"role": "user", "content": content}] + svc.thinking = thinking + return svc.chat(msgs) + + submit_btn.click(infer, [model_dd, thinking_cb, image_in, video_in, prompt_in], output_txt) +``` + +--- + +## 12. Field Notes & Correction Loop + +```python +# datasets/field_notes.py +import sqlite3, json +from dataclasses import dataclass, field, asdict +from datetime import datetime + +@dataclass +class FieldNote: + id: str = field(default_factory=lambda: datetime.utcnow().isoformat()) + model_id: str = "" + modality: str = "text" # "text" | "image" | "video" | "multimodal" + image_path: str | None = None + video_path: str | None = None + prompt: str = "" + response: str = "" + correction: str = "" # human-corrected output + tags: list = field(default_factory=list) + used_in_train: bool = False + +class FieldNoteStore: + def __init__(self, db_path: str = "data/field_notes.db"): + self.conn = sqlite3.connect(db_path, check_same_thread=False) + self._init_db() + + def _init_db(self): + self.conn.execute(""" + CREATE TABLE IF NOT EXISTS notes ( + id TEXT PRIMARY KEY, + data JSON NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + self.conn.commit() + + def save(self, note: FieldNote): + self.conn.execute( + "INSERT OR REPLACE INTO notes (id, data) VALUES (?, ?)", + (note.id, json.dumps(asdict(note))) + ) + self.conn.commit() + + def to_hf_dataset(self, output_path: str): + """Export uncorrected notes as HF Dataset for retraining.""" + from datasets import Dataset + rows = [json.loads(r[0]) for r in + self.conn.execute("SELECT data FROM notes WHERE json_extract(data,'$.correction') != ''")] + ds = Dataset.from_list(rows) + ds.save_to_disk(output_path) + return ds +``` + +--- + +## 13. Directory Structure + +``` +ml-workbench/ +├── app.py # Entry point: gr.Blocks or gradio.Server +├── requirements.txt +│ +├── config/ +│ ├── models.yaml # ← TEMPLATE AXIS: change this for any model family +│ └── training.yaml # LoRA hyperparams, export settings +│ +├── core/ +│ ├── events.py # EventBus, EventType enum +│ ├── registry.py # Registry[T] generic +│ └── state.py # Global AppState dataclass +│ +├── models/ +│ ├── base.py # Abstract ModelService +│ ├── hf_loader.py # Generic HF Hub downloader +│ ├── minicpm_text.py # MiniCPM5-1B, MiniCPM4.1-8B service +│ ├── minicpm_vision.py # MiniCPM-V-4.6 / Thinking service +│ ├── minicpm_omni.py # MiniCPM-o-4.5 service +│ ├── llama_cpp_runner.py # llama-server wrapper +│ ├── sglang_runner.py # SGLang server wrapper (tool-use) +│ └── ollama_runner.py # Ollama client +│ +├── datasets/ +│ ├── loader.py # Abstract DatasetLoader +│ ├── hf_datasets.py # HF Hub datasets.load_dataset() +│ ├── field_notes.py # SQLite-backed correction store +│ └── synthetic.py # ml-intern style synthetic data gen +│ +├── training/ +│ ├── lora.py # LoRA text trainer (PEFT + TRL) +│ ├── lora_vision.py # LoRA vision config (SWIFT/LLaMA-Factory) +│ ├── evaluation.py # Perplexity, accuracy, custom metrics +│ ├── export.py # GGUF exporter (convert_hf_to_gguf.py) +│ └── hf_jobs.py # HF Jobs offload (ml-intern pattern) +│ +├── tools/ +│ ├── calculator.py # Safe math eval tool +│ ├── dataset_stats.py # Dataset statistics tool +│ ├── hf_search.py # HF Hub search tool +│ └── paper_search.py # arXiv / HF Papers search (ml-intern) +│ +├── tracking/ +│ └── trackio_service.py # Trackio integration (init/log/finish/alert) +│ +├── mcp/ +│ ├── server.py # FastMCP standalone server +│ └── tools.py # All @mcp.tool() decorators +│ +├── agent/ +│ ├── loop.py # Research → Plan → Implement agent +│ └── prompts/ +│ └── system_prompt.yaml # Agent system prompt (ml-intern style) +│ +├── ui/ +│ ├── chat_tab.py +│ ├── vision_tab.py +│ ├── train_tab.py +│ ├── dataset_tab.py +│ ├── export_tab.py +│ ├── traces_tab.py +│ ├── agent_tab.py +│ └── notes_tab.py +│ +├── exports/ # GGUF + quantized model outputs +└── data/ # Cached datasets, field_notes.db +``` + +--- + +## 14. Configuration Schema + +### training.yaml + +```yaml +# config/training.yaml +lora: + rank: 16 + alpha: 32 + dropout: 0.05 + target_modules_text: + - q_proj + - k_proj + - v_proj + - o_proj + - gate_proj + - up_proj + - down_proj + target_modules_vision: + - q_proj + - k_proj + - v_proj + - o_proj # language layers only; vision tower frozen + +training: + epochs: 3 + batch_size: 4 + grad_accum: 4 + lr: 2.0e-4 + bf16: true + report_to: trackio # or: wandb, tensorboard, none + +export: + default_quant: Q4_K_M + supported_quants: + - F16 + - Q4_K_M + - Q5_K_M + - Q8_0 + - Q2_K + +trackio: + project: ml-workbench + space_id: null # set to "username/trackio-space" for HF sync + +hf_jobs: + enabled: false + hardware: gpu-a100 +``` + +--- + +## 15. Dependencies + +```text +# requirements.txt +# Core +gradio[mcp]>=5.38.0 +mcp[cli]>=1.0.0 +fastapi +uvicorn[standard] + +# Models +transformers>=5.7.0 # required for MiniCPM-V-4.6 +torch>=2.1.0 +torchvision +torchcodec # video decoding for MiniCPM-V-4.6 +accelerate +bitsandbytes + +# Training +peft>=0.10.0 +trl>=0.9.0 +datasets>=2.18.0 + +# Tracking +trackio>=0.1.0 + +# MCP + Agents +mcp>=1.0.0 +smolagents>=1.0.0 + +# Serving backends (optional, install as needed) +# sglang # pip install sglang +# vllm # pip install vllm +# ollama # install separately + +# Hub +huggingface_hub>=0.23.0 +Pillow +requests +pyyaml +``` + +--- + +## 16. Hackathon Demo Flow + +``` +1. DATASET + Load HF dataset via Dataset tab (e.g. "tatsu-lab/alpaca", 10K rows) + → EventType.DATASET_LOADED + → Trackio: trackio.init("demo-run") + trackio.log({"dataset": ..., "rows": ...}) + +2. TRAIN + Start LoRA on MiniCPM5-1B (rank=16, 1 epoch demo) + → EventType.TRAINING_STARTED + → TRL SFTTrainer + report_to="trackio" → live loss in Traces tab + +3. EXPORT + Click "Export GGUF" → Q4_K_M + → convert_hf_to_gguf.py → llama-quantize + → EventType.EXPORT_FINISHED → file download + +4. LLAMA.CPP CHAT + Load exported GGUF in llama-server + → Chat tab, backend: llama.cpp + → Verify text generation quality + +5. SWITCH TO VISION + Select MiniCPM-V-4.6 in Vision tab + Upload image → enter prompt → Run + +6. THINKING MODE + Toggle "Enable Thinking Mode" → switch to MiniCPM-V-4.6-Thinking + → Observe explicit reasoning trace in response + +7. FIELD NOTE + Response is wrong → open Field Notes tab + → Enter correction → Save + → EventType.FIELD_NOTE_SAVED + +8. AGENT MODE + Agent tab: "Improve MiniCPM-V-4.6 on [domain] using field notes" + → Research phase: browse HF Papers + → Plan: dataset selection, LoRA config + → Implement: trigger SWIFT fine-tune + +9. SHARE TRACES + Traces tab → set space_id → Sync to HF Space + → Shareable dashboard URL +``` + +--- + +## 17. Corrections from PRD v1 + +| v1 Claim | Correct v2 | +|----------|-----------| +| `trackio.trace("vision_inference")` context manager | **Does not exist.** Use `trackio.init()` + `trackio.log()` + `trackio.finish()` | +| `AutoModelForCausalLM` for MiniCPM-V-4.6 | **Wrong.** Use `AutoModelForImageTextToText` with `transformers>=5.7.0` | +| `llama-mtmd-cli` for vision deployment | Use `llama-server --mmproj ` for server; `llama-mtmd-cli` for CLI-only | +| Run `convert_hf_to_gguf.py` for V4.6 | Official GGUFs at `openbmb/MiniCPM-V-4.6-gguf` — download directly; run script only for custom quants | +| Surgery scripts for V4.6 GGUF | Surgery scripts are for MiniCPM-o-2.6 only; **do not use for V4.6** | +| `future MiniCPM-o` (vague) | **MiniCPM-o-4.5** released 2026-05-17; specific HF ID available | +| `app.launch(mcp_server=True)` only option | Two paths: `gr.Blocks` + `mcp_server=True`, or `gradio.Server` with `@app.mcp.tool()` | +| FastMCP as only MCP approach | Three paths: Gradio native, gradio.Server, standalone FastMCP (see §7) | +| MiniCPM4.1-8B: standard architecture | Has sparse InfLLM-v2 attention; requires `--trust-remote-code` | +| LoRA for vision: PEFT only | OpenBMB recommends **SWIFT** or **LLaMA-Factory** for MiniCPM-V fine-tuning | +| No HF Jobs mention | HF Jobs integration (ml-intern pattern) enables GPU cloud offload | +| No SGLang mention | SGLang is the **recommended** backend for MiniCPM5-1B tool calling | +| No video support | MiniCPM-V-4.6 supports **video understanding** (`torchcodec` required) | + +--- + +## 18. Roadmap & Extension Points + +### Swap the model family (template portability) + +To build a "Qwen3 Workbench": +1. Edit `config/models.yaml` — add Qwen3 model IDs +2. Create `models/qwen3_service.py` extending `ModelService` +3. Register in `app.py` — done + +No changes to training, export, MCP, Trackio, or UI layers. + +### Planned extensions + +| Feature | Module | Notes | +|---------|--------|-------| +| vLLM serving tab | `models/vllm_runner.py` | Batch inference, production | +| Ollama quick-start | `models/ollama_runner.py` | Zero-config consumer demo | +| Reward model eval | `training/reward_eval.py` | For RLHF experiments | +| Synthetic data gen | `datasets/synthetic.py` | ml-intern pattern: LLM writes training data | +| Paper-to-code agent | `agent/paper_agent.py` | Read arXiv → implement in workbench | +| HF Spaces deploy | `deploy/spaces.py` | One-click push as HF Space | +| VINDEX integration | `tools/vindex_tool.py` | Knowledge editing via your VINDEX engine | +| OCR pipeline hook | `datasets/ocr_loader.py` | Feed your OCR pipeline output as field notes | +| MiniCPM Desk-Pet | `agent/desk_pet.py` | LoRA persona switching via MiniCPM5-1B | +| MiniCPM-o audio | `ui/audio_tab.py` | Real-time omnimodal via MiniCPM-o-4.5 | + +--- + +*PRD v2.0 — Christof Kaller / ki-fusion-labs.de — 2026-06-05* +*Research sources: openbmb/MiniCPM-V, openbmb/MiniCPM, gradio-app/trackio,* +*huggingface/ml-intern, huggingface.co/blog/introducing-gradio-server* \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5a2fa7049c1776afd8b0cfa9c554976257963e84 --- /dev/null +++ b/README.md @@ -0,0 +1,227 @@ +--- +title: OpenBMB Local AI Workbench +colorFrom: green +colorTo: gray +sdk: gradio +app_file: app.py +pinned: false +--- + +# OpenBMB Local AI Workbench + +A Gradio workbench for the Build Small Hackathon: small local models, practical experimentation, +and a clear path from local demo to Hugging Face Space. + +## What This Is + +The project turns the PRD in `HF_PRD_v1.md` into a staged implementation: + +1. A working Gradio app shell. +2. Config-driven OpenBMB model registry. +3. Local-first inference path through real backends: Transformers, Ollama, llama.cpp, + LM Studio/OpenAI-compatible, SGLang, and vLLM. +4. Field notes for collecting corrections. +5. Extension points for training, GGUF export, Trackio traces, MCP tools, and agent workflows. + +## Hackathon Fit + +- **Track:** Backyard AI or Thousand Token Wood, depending on the final user story. +- **Canvas:** Gradio app, deployable to Hugging Face Spaces. +- **Small model rule:** target models stay at or below 32B parameters. +- **Bonus quests:** local-first, field notes/report, possible llama.cpp and trace sharing. + +## Quick Start + +Python is not currently available on PATH in this workspace shell. Install Python first, then: + +```powershell +.\scripts\verify_structure.ps1 +& "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" -m venv .venv +.venv\Scripts\Activate.ps1 +python -m pip install -r requirements.txt +python -m pip install -r requirements-dev.txt +.\scripts\run_tests.ps1 +python app.py +``` + +## Browser E2E Screenshots + +After installing Node.js: + +```powershell +npm install +npm run e2e:install +npm run e2e +``` + +The Playwright user-story tests run separate Workbench and Plant flows and save documentation +screenshots under `assets/e2e/workbench/` and `assets/e2e/plant/`. The Workbench screenshot now runs +local GGUF chat through `llama-cpp-python`; the Plant screenshot can run OpenBMB MiniCPM-V on +`assets/plant_sample.jpg` with `RUN_REAL_MODEL_E2E=1`. +only when `RUN_REAL_MODEL_E2E=1` is set; otherwise the browser tests verify real-backend setup +surfaces without using mock responses. To record or edit the browser flow manually, run +`npm run e2e:record`. + +Generated screenshot sets: + +- [Workbench home](assets/e2e/workbench/01-workbench-home.png) +- [Workbench backend status](assets/e2e/workbench/05-backend-status.png) +- [Plant tool home](assets/e2e/plant/01-plant-home.png) +- [Plant corrections export](assets/e2e/plant/03-corrections-export.png) + +## Template And Reference Apps + +This repo is also a template for focused local-first Gradio apps. The first reference app is +Plant Discovery under `plant/`. + +```powershell +.venv\Scripts\python.exe -m plant.app --no-model --port 7861 +``` + +Use the real OpenBMB VLM path after installing optional plant dependencies: + +```powershell +.venv\Scripts\python.exe -m plant.app --model-mode openbmb --port 7861 +``` + +The detailed build guide is [docs/TEMPLATE_HOWTO.md](docs/TEMPLATE_HOWTO.md), and the Plant +Discovery checklist is [docs/PLANT_DISCOVERY_APP_PLAN.md](docs/PLANT_DISCOVERY_APP_PLAN.md). +Model and adapter training steps are in +[docs/PLANT_MODEL_AND_TRAINING_HOWTO.md](docs/PLANT_MODEL_AND_TRAINING_HOWTO.md). + +If `python` is not recognized, install Python 3.11+ from python.org or the Windows Store, +then reopen the terminal so PATH is refreshed. + +Open the local URL shown by Gradio, usually `http://127.0.0.1:7860`. + +## Project Structure + +```text +. +|-- app.py +|-- AGENTS.md +|-- README.md +|-- requirements.txt +|-- config/ +| |-- models.yaml +| `-- training.yaml +|-- core/ +| |-- events.py +| `-- registry.py +|-- datasets/ +| `-- field_notes.py +|-- models/ +| |-- placeholder_service.py +| `-- model_catalog.py +|-- ui/ +| |-- chat_tab.py +| |-- dataset_tab.py +| |-- export_tab.py +| |-- notes_tab.py +| |-- traces_tab.py +| |-- agent_tab.py +| |-- status_tab.py +| |-- train_tab.py +| `-- vision_tab.py +|-- data/ +| `-- .gitkeep +|-- exports/ +| `-- .gitkeep +|-- HF_PRD_v1.md +`-- HF_PRD_ext.md +``` + +## Project Docs + +The working docs live in [docs/README.md](docs/README.md). + +- [Task checklist](docs/TASKS.md) +- [Implementation status](docs/IMPLEMENTATION_STATUS.md) +- [PRD implementation matrix](docs/PRD_IMPLEMENTATION_MATRIX.md) +- [Acceptance criteria](docs/ACCEPTANCE_CRITERIA.md) +- [Roadmap](docs/ROADMAP.md) +- [Template how-to](docs/TEMPLATE_HOWTO.md) +- [Plant Discovery plan](docs/PLANT_DISCOVERY_APP_PLAN.md) +- [Usage guide](docs/USAGE.md) +- [Architecture guide](docs/ARCHITECTURE.md) +- [Extension guide](docs/EXTENDING.md) +- [Hackathon submission package](docs/HACKATHON_SUBMISSION.md) +- [Test and quality policy](docs/ACCEPTANCE_CRITERIA.md) + +## Current Truth + +The full PRD is not implemented yet. The current app is a tested, quality-gated scaffold moving +from placeholder-first local verification to real-backend Workbench deployment. GitHub push is +complete at `https://github.com/Ckal/codex`. LM Studio/OpenAI-compatible text inference has been +verified previously; OpenBMB Transformers, Ollama OpenBMB, llama.cpp, MiniCPM-V, Space builds, +training execution, served MCP, and most extension PRD items still need proof before being claimed +done. + +## Model Plan + +Initial candidates from the PRD: + +| Config ID | Model | Purpose | +| --- | --- | --- | +| `minicpm5_1b` | `openbmb/MiniCPM5-1B` | text chat, LoRA, local-first baseline | +| `minicpm_v46` | `openbmb/MiniCPM-V-4.6` | image/video understanding | +| `minicpm_o45` | `openbmb/MiniCPM-o-4.5` | omnimodal stretch goal | + +The app does not download large model files automatically. In deployed Space mode, placeholder +backend choices are hidden and model calls require real backend configuration. llama.cpp, +llama-cpp-python, Ollama, LM Studio/OpenAI-compatible, SGLang, Nemotron Nano 9B v2, and Transformers text can be selected as +backends, but the backend tool/package/server must be installed and populated with the selected +model explicitly by the user. + +## Deployment Target + +For Hugging Face Spaces, keep these files at repo root: + +- `app.py` +- `requirements.txt` +- `README.md` +- `config/` +- `core/` +- `models/` +- `ui/` + +Workbench Space target: + +```text +https://huggingface.co/spaces/build-small-hackathon/workbench +``` + +## Spaces + +- Workbench Space: https://huggingface.co/spaces/build-small-hackathon/workbench +- Plant Identification Tool Space: https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool + +Both Spaces have been pushed. At the latest local poll they were still in Hugging Face `BUILDING` +state on `zero-a10g`, so final build/run smoke verification is still open. + +Plant Identification Tool Space target: + +```text +https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool +``` + +Use a freshly generated token through `hf auth login`; do not paste tokens into files or commit +them. + +Later deployment commands: + +```powershell +hf auth login +git remote add space-workbench https://huggingface.co/spaces/build-small-hackathon/workbench +git push space-workbench main +git remote add space-plant https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool +git push space-plant main +``` + +## Next Implementation Steps + +1. Decide the exact hackathon story and user. +2. Add screenshot/demo media and Space submission URLs. +3. Push and verify the two Hugging Face Spaces, then finish llama.cpp MiniCPM-V mmproj vision verification. +4. Add field-note export to JSONL/HF Dataset. +5. Polish README with screenshots, demo video script, and submission links. diff --git a/agent/__init__.py b/agent/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..45664997acb704a486c360f6ac75e2ace59d1466 --- /dev/null +++ b/agent/__init__.py @@ -0,0 +1 @@ +"""Local deterministic agent helpers.""" diff --git a/agent/runner.py b/agent/runner.py new file mode 100644 index 0000000000000000000000000000000000000000..ef56517a158f2343ece8924ef7f0159e2f9155c6 --- /dev/null +++ b/agent/runner.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from core.file_exports import copy_text_file_or_empty +from mcp_tools.tools import safe_calculator_tool, tool_registry + +AGENT_SYSTEM_PROMPT = ( + "You are a local workbench agent. Research the request, draft a small plan, " + "name tools you would use, and require verification before marking work done." +) + + +@dataclass(frozen=True) +class AgentStep: + """One deterministic agent trace step.""" + + phase: str + content: str + + +@dataclass(frozen=True) +class AgentSession: + """Agent session trace.""" + + task: str + steps: list[AgentStep] + tools: list[str] + limitations: list[str] + safety_gates: list[str] + + def as_dict(self) -> dict[str, Any]: + return { + "task": self.task, + "steps": [asdict(step) for step in self.steps], + "tools": self.tools, + "limitations": self.limitations, + "safety_gates": self.safety_gates, + "system_prompt": AGENT_SYSTEM_PROMPT, + } + + def as_markdown(self) -> str: + lines = [f"Task: {self.task or '(none)'}", ""] + for step in self.steps: + lines.append(f"{step.phase}: {step.content}") + lines.append("") + lines.append(f"Tools: {', '.join(self.tools)}") + lines.append(f"Limitations: {'; '.join(self.limitations)}") + lines.append(f"Safety gates: {'; '.join(self.safety_gates)}") + return "\n".join(lines) + + +def run_agent_loop(task: str) -> AgentSession: + tools = sorted(tool_registry()) + steps = [ + AgentStep("research", _research_summary(task)), + AgentStep("plan", _plan_summary(task)), + AgentStep("implement", _implementation_summary(task)), + AgentStep("verify", "Run unit tests, smoke checks, quality gates, and update docs/tasks."), + ] + + calculator_result = _maybe_calculate(task) + if calculator_result is not None: + steps.insert( + 1, + AgentStep( + "tool:safe_calculator", + json.dumps(calculator_result.payload, ensure_ascii=False), + ), + ) + + return AgentSession( + task=task, + steps=steps, + tools=tools, + limitations=[ + "Does not execute shell commands.", + "Does not commit, push, deploy, download models, or call external services.", + "Requires Codex or a human to apply and verify implementation changes.", + ], + safety_gates=default_safety_gates(), + ) + + +def run_paper_to_code_loop( + paper_title: str, + paper_notes: str, + implementation_goal: str, +) -> AgentSession: + task = f"Paper-to-code: {paper_title.strip() or 'untitled paper'}" + steps = [ + AgentStep("research", _paper_research_summary(paper_title, paper_notes)), + AgentStep("plan", _paper_plan_summary(implementation_goal)), + AgentStep("implement", _paper_implementation_trace(implementation_goal)), + AgentStep("verify", "Map claims to tests, run quality gates, and document gaps."), + ] + return AgentSession( + task=task, + steps=steps, + tools=sorted(tool_registry()), + limitations=[ + "Does not read remote papers automatically.", + "Does not execute code changes autonomously.", + "Requires human/Codex review before implementation claims are marked done.", + ], + safety_gates=default_safety_gates(), + ) + + +def default_safety_gates() -> list[str]: + return [ + "No shell commands are executed by the agent trace.", + "No model weights, datasets, or papers are downloaded automatically.", + "Every implementation claim needs a matching test or documented blocker.", + "External services require explicit user credentials and approval.", + ] + + +def save_agent_trace( + session: AgentSession, + path: str | Path = "data/agent_traces.jsonl", +) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("a", encoding="utf-8") as f: + f.write(json.dumps(session.as_dict(), ensure_ascii=False) + "\n") + return output + + +def export_agent_traces( + source_path: str | Path = "data/agent_traces.jsonl", + output_path: str | Path = "exports/agent_traces.jsonl", +) -> Path: + return copy_text_file_or_empty(source_path, output_path) + + +def export_agent_traces_hf_dataset( + source_path: str | Path = "data/agent_traces.jsonl", + output_dir: str | Path = "exports/agent_traces_dataset", +) -> Path: + target = Path(output_dir) + target.mkdir(parents=True, exist_ok=True) + data_file = target / "data.jsonl" + if Path(source_path).exists(): + data_file.write_text(Path(source_path).read_text(encoding="utf-8"), encoding="utf-8") + else: + data_file.write_text("", encoding="utf-8") + (target / "README.md").write_text( + "# Agent Traces Dataset\n\n" + "Local Hugging Face Dataset-style export for OpenBMB Local AI Workbench traces.\n", + encoding="utf-8", + ) + return target + + +def _research_summary(task: str) -> str: + if not task.strip(): + return "No task provided. Ask for a concrete task before implementation." + return "Inspect PRD/tasks/docs, identify affected modules, and check existing tests." + + +def _plan_summary(task: str) -> str: + if any(word in task.casefold() for word in ["deploy", "push", "github", "huggingface"]): + return "Prepare repo/deploy steps, verify auth/remotes, then push only after tests pass." + return "Make a focused implementation slice, add or update tests, then update docs." + + +def _implementation_summary(task: str) -> str: + if "model" in task.casefold(): + return "Use configured backend services and avoid startup downloads." + return "Apply changes in the smallest relevant modules and keep unrelated files untouched." + + +def _paper_research_summary(paper_title: str, paper_notes: str) -> str: + title = paper_title.strip() or "untitled paper" + notes = paper_notes.strip() + if not notes: + return f"Summarize the claims, assumptions, and reproducibility risks for {title}." + return f"Extract implementation claims from {title}: {notes[:240]}" + + +def _paper_plan_summary(implementation_goal: str) -> str: + goal = implementation_goal.strip() or "create a minimal local reproduction plan" + return f"Break the goal into local modules, tests, data assumptions, and blockers: {goal}." + + +def _paper_implementation_trace(implementation_goal: str) -> str: + goal = implementation_goal.strip() or "minimal reproducible scaffold" + return ( + "Draft a non-executing implementation trace for " + f"{goal}; keep dependencies explicit and update docs before claiming completion." + ) + + +def _maybe_calculate(task: str): + prefix = "calculate:" + if task.casefold().strip().startswith(prefix): + expression = task.split(":", 1)[1].strip() + return safe_calculator_tool(expression) + return None diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..64aba11176ca8e666b2638e1b24265e999691d73 --- /dev/null +++ b/app.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import cast + +import gradio as gr + +from core.app_logging import configure_app_logging +from core.deployment import current_policy +from models.model_catalog import load_model_catalog +from ui.agent_tab import build_agent_tab +from ui.chat_tab import build_chat_tab +from ui.dataset_tab import build_dataset_tab +from ui.export_tab import build_export_tab +from ui.notes_tab import build_notes_tab +from ui.status_tab import build_status_tab +from ui.traces_tab import build_traces_tab +from ui.train_tab import build_train_tab +from ui.vision_tab import build_vision_tab +from ui.vllm_tab import build_vllm_tab + +APP_THEME = gr.themes.Soft(primary_hue="teal", neutral_hue="slate") +APP_CSS = """ +.app-title { + margin-bottom: 0.25rem; +} + +.gradio-container { + max-width: 1180px !important; +} + +.tabs { + overflow-x: auto; +} + +button { + min-height: 2.5rem; +} + +textarea, +input, +.wrap { + max-width: 100%; +} + +@media (max-width: 720px) { + .gradio-container { + padding-left: 0.75rem !important; + padding-right: 0.75rem !important; + } + + .app-title h1 { + font-size: 1.55rem; + line-height: 1.2; + } +} +""" + + +def build_app() -> gr.Blocks: + configure_app_logging() + policy = current_policy() + catalog = load_model_catalog("config/models.yaml") + + with gr.Blocks(title="OpenBMB Local AI Workbench", analytics_enabled=False) as demo: + gr.Markdown( + """ + # OpenBMB Local AI Workbench + Real small-model experimentation for Gradio + Hugging Face Spaces. + """, + elem_classes=["app-title"], + ) + if policy.is_space: + gr.Markdown( + "Deployment mode: Space. Placeholder and demo inference are disabled; " + "select a configured real backend before running model calls." + ) + + with gr.Tabs(): + with gr.Tab("Chat"): + build_chat_tab(catalog, policy) + with gr.Tab("Vision"): + build_vision_tab(catalog, policy) + with gr.Tab("Dataset"): + build_dataset_tab() + with gr.Tab("Train"): + build_train_tab() + with gr.Tab("vLLM"): + build_vllm_tab(catalog) + with gr.Tab("Export"): + build_export_tab(catalog) + with gr.Tab("Field Notes"): + build_notes_tab(catalog) + with gr.Tab("Traces"): + build_traces_tab() + with gr.Tab("Agent"): + build_agent_tab() + with gr.Tab("Status"): + build_status_tab(catalog, policy) + + return cast(gr.Blocks, demo) + + +if __name__ == "__main__": + build_app().launch( + server_port=7860, + share=False, + theme=APP_THEME, + css=APP_CSS, + mcp_server=True, + ) diff --git a/assets/e2e/README.md b/assets/e2e/README.md new file mode 100644 index 0000000000000000000000000000000000000000..28d8940b573b5a5754b2c8b1ccb85e7366abeed3 --- /dev/null +++ b/assets/e2e/README.md @@ -0,0 +1,21 @@ +# E2E Documentation Assets + +Run the Playwright user-story test to generate screenshots for the README and hackathon docs: + +```powershell +npm install +npm run e2e:install +npm run e2e +``` + +The test starts or reuses the local Gradio app at `http://127.0.0.1:7860`, walks all major tabs, +executes safe local actions, and writes screenshots plus `user-story.md` into this folder. + +To record or edit the flow manually: + +```powershell +npm run e2e:record +``` + +Generated images are intentionally stored under `assets/e2e/` so they can be linked from +`README.md`, `docs/HACKATHON_SUBMISSION.md`, and demo notes. diff --git a/assets/e2e/plant-playwright-report/index.html b/assets/e2e/plant-playwright-report/index.html new file mode 100644 index 0000000000000000000000000000000000000000..7dba4a8a6809ad3915ebd76b0a4fe5c9a71986c3 --- /dev/null +++ b/assets/e2e/plant-playwright-report/index.html @@ -0,0 +1,90 @@ + + + + + + + + + Playwright Test Report + + + + +
+ + + \ No newline at end of file diff --git a/assets/e2e/playwright-report/index.html b/assets/e2e/playwright-report/index.html new file mode 100644 index 0000000000000000000000000000000000000000..0a098831bba3038845cd7e5547f00f3a252fec60 --- /dev/null +++ b/assets/e2e/playwright-report/index.html @@ -0,0 +1,90 @@ + + + + + + + + + Playwright Test Report + + + + +
+ + + \ No newline at end of file diff --git a/assets/samples/README.md b/assets/samples/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bbc85feb1ecf403558a1142ab3e572b39580113f --- /dev/null +++ b/assets/samples/README.md @@ -0,0 +1,7 @@ +# Public Sample Assets + +These files are intentionally small and public-safe. They are used for local verification, +Playwright screenshots, and documentation examples. + +- `plant_leaf.ppm` and `plant_leaf.png` are generated bitmap-style plant samples for upload workflow testing. + It is not a botanical benchmark image and should not be used to claim model accuracy. diff --git a/assets/samples/plant_leaf.png b/assets/samples/plant_leaf.png new file mode 100644 index 0000000000000000000000000000000000000000..93ca0fb0da2de8a2baacda7185b7d7f9f9f356ce Binary files /dev/null and b/assets/samples/plant_leaf.png differ diff --git a/assets/samples/plant_leaf.ppm b/assets/samples/plant_leaf.ppm new file mode 100644 index 0000000000000000000000000000000000000000..c63ca68ac65857a0050caaab608f436f5e215dcc --- /dev/null +++ b/assets/samples/plant_leaf.ppm @@ -0,0 +1,11 @@ +P3 +8 8 +255 +255 255 255 235 250 235 220 245 220 205 238 205 205 238 205 220 245 220 235 250 235 255 255 255 +245 252 245 210 240 210 130 185 130 70 145 70 70 145 70 130 185 130 210 240 210 245 252 245 +235 250 235 160 210 160 60 140 60 30 110 30 30 110 30 60 140 60 160 210 160 235 250 235 +225 245 225 120 190 120 40 125 40 25 95 25 25 95 25 40 125 40 120 190 120 225 245 225 +225 245 225 120 190 120 40 125 40 25 95 25 25 95 25 40 125 40 120 190 120 225 245 225 +235 250 235 160 210 160 60 140 60 30 110 30 30 110 30 60 140 60 160 210 160 235 250 235 +245 252 245 210 240 210 130 185 130 70 145 70 70 145 70 130 185 130 210 240 210 245 252 245 +255 255 255 235 250 235 220 245 220 205 238 205 205 238 205 220 245 220 235 250 235 255 255 255 diff --git a/config/models.yaml b/config/models.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de9061d130f194f1b74c4f9a87004d900950876b --- /dev/null +++ b/config/models.yaml @@ -0,0 +1,155 @@ +models: + minicpm5_1b: + hf_id: openbmb/MiniCPM5-1B + display_name: MiniCPM5 1B + type: text + parameters_b: 1 + backend: transformers + backend_capabilities: + placeholder: [text] + ollama: [text, streaming] + llama.cpp: [text, gguf] + llama-cpp-python: [text, gguf] + transformers: [text, lora] + openai-compatible: [text, openai-compatible] + sglang: [text, tools] + vllm: [text, openai-compatible] + context_length: 131072 + local_first: true + thinking_mode: false + gguf: + available: true + repo: openbmb/MiniCPM5-1B-GGUF + default_quant: Q4_K_M + notes: Text baseline for local chat, LoRA experiments, and GGUF export. + + minicpm5_1b_thinking: + hf_id: openbmb/MiniCPM5-1B-Thinking + display_name: MiniCPM5 1B Thinking + type: text + parameters_b: 1 + backend: transformers + backend_capabilities: + placeholder: [text] + ollama: [text, thinking, streaming] + llama.cpp: [text, gguf] + llama-cpp-python: [text, gguf] + transformers: [text, thinking, lora] + openai-compatible: [text, thinking, openai-compatible] + sglang: [text, thinking, tools] + vllm: [text, openai-compatible] + context_length: 131072 + local_first: true + thinking_mode: true + gguf: + available: true + repo: openbmb/MiniCPM5-1B-Thinking-GGUF + default_quant: Q4_K_M + notes: Thinking variant for explicit reasoning experiments. + + minicpm41_8b: + hf_id: openbmb/MiniCPM4.1-8B + display_name: MiniCPM4.1 8B + type: text + parameters_b: 8 + backend: transformers + backend_capabilities: + placeholder: [text] + ollama: [text, streaming] + llama.cpp: [text, gguf] + llama-cpp-python: [text, gguf] + transformers: [text, long-context] + openai-compatible: [text, long-context, openai-compatible] + sglang: [text, long-context, tools] + vllm: [text, long-context, openai-compatible] + context_length: 131072 + local_first: true + thinking_mode: false + trust_remote_code: true + gguf: + available: true + repo: openbmb/MiniCPM4.1-8B-GGUF + default_quant: Q4_K_M + notes: Long-context sparse-attention text model; real backend will require trust_remote_code. + + nemotron_nano_9b_v2: + hf_id: nvidia/NVIDIA-Nemotron-Nano-9B-v2 + display_name: NVIDIA Nemotron Nano 9B v2 + type: text + parameters_b: 9 + backend: transformers + backend_capabilities: + placeholder: [text] + llama.cpp: [text, gguf] + llama-cpp-python: [text, gguf] + transformers: [text, custom-code] + openai-compatible: [text, openai-compatible] + sglang: [text] + vllm: [text, openai-compatible] + context_length: 131072 + local_first: true + thinking_mode: false + trust_remote_code: true + gguf: + available: true + repo: bartowski/nvidia_NVIDIA-Nemotron-Nano-9B-v2-GGUF + default_quant: Q4_K_M + notes: Small Nemotron text option under the 32B hackathon limit; verify license and hardware before using in a public Space. + + minicpm_v46: + hf_id: openbmb/MiniCPM-V-4.6 + display_name: MiniCPM-V 4.6 + type: vision + parameters_b: 1.3 + backend: transformers + backend_capabilities: + placeholder: [vision] + ollama: [vision] + llama.cpp: [vision, gguf, mmproj] + transformers: [vision, image-text-to-text] + sglang: [vision, openai-compatible] + context_length: 262144 + local_first: true + thinking_mode: false + gguf: + available: true + repo: openbmb/MiniCPM-V-4.6-gguf + main_file: MiniCPM-V-4.6-Q4_K_M.gguf + mmproj: mmproj-MiniCPM-V-4.6-F16.gguf + default_quant: Q4_K_M + notes: Vision-language model candidate for image and video understanding. + + minicpm_v46_thinking: + hf_id: openbmb/MiniCPM-V-4.6-Thinking + display_name: MiniCPM-V 4.6 Thinking + type: vision + parameters_b: 1.3 + backend: transformers + backend_capabilities: + placeholder: [vision, thinking] + ollama: [vision, thinking] + llama.cpp: [vision, gguf, mmproj] + transformers: [vision, thinking, image-text-to-text] + sglang: [vision, thinking, openai-compatible] + context_length: 262144 + local_first: true + thinking_mode: true + gguf: + available: true + repo: openbmb/MiniCPM-V-4.6-Thinking-gguf + default_quant: Q4_K_M + notes: Vision-language thinking variant for multimodal reasoning demos. + + minicpm_o45: + hf_id: openbmb/MiniCPM-o-4.5 + display_name: MiniCPM-o 4.5 + type: omnimodal + parameters_b: 8 + backend: transformers + backend_capabilities: + placeholder: [text, vision, audio] + transformers: [text, vision, audio] + context_length: 131072 + local_first: true + thinking_mode: false + notes: Omnimodal stretch goal for audio, image, and text. diff --git a/config/training.yaml b/config/training.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b15320cf2b56f89df3fc2fa3da9f39e31dba3d4 --- /dev/null +++ b/config/training.yaml @@ -0,0 +1,24 @@ +lora: + rank: 16 + alpha: 32 + dropout: 0.05 + +training: + epochs: 1 + batch_size: 2 + grad_accum: 4 + lr: 0.0002 + report_to: none + +export: + default_quant: Q4_K_M + supported_quants: + - F16 + - Q4_K_M + - Q5_K_M + - Q8_0 + +trackio: + enabled: false + project: openbmb-local-ai-workbench + local_path: data/traces.jsonl diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f3e951a03e69d713647a221ddc0605a8c7bb86ec --- /dev/null +++ b/core/__init__.py @@ -0,0 +1 @@ +"""Core shared helpers for events, state, and registries.""" diff --git a/core/app_logging.py b/core/app_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..808706d786c20509259b0d4a1b61d5bb9a5398d2 --- /dev/null +++ b/core/app_logging.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import logging + + +def configure_app_logging() -> None: + """Configure compact application logging once.""" + + if logging.getLogger().handlers: + return + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) diff --git a/core/app_state.py b/core/app_state.py new file mode 100644 index 0000000000000000000000000000000000000000..d13cc0c7f23119a051044a5e98ae71ac54e0a36a --- /dev/null +++ b/core/app_state.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import logging +from dataclasses import asdict + +from core.events import Event, EventBus, EventType +from tracking.trackio_client import TrackingClient + + +class AppState: + """Shared in-process state for UI events and lightweight traces.""" + + def __init__( + self, + event_bus: EventBus | None = None, + tracking_client: TrackingClient | None = None, + ) -> None: + self.event_bus = event_bus or EventBus() + self.events: list[Event] = [] + self.logger = logging.getLogger("openbmb_workbench") + self.tracking_client = tracking_client or TrackingClient() + self.tracking_client.init() + + def emit(self, event: Event) -> None: + self.events.append(event) + self.logger.info("event=%s payload=%s", event.type.value, event.payload) + self.tracking_client.log(event.type.value, event.payload) + self.event_bus.emit(event) + + def recent_events(self, limit: int = 20) -> list[dict]: + recent = self.events[-limit:] + return [asdict(event) for event in recent] + + +def emit_inference_response( + mode: str, + model_id: str, + backend: str, + response: str, + state: AppState | None = None, +) -> None: + target_state = state or APP_STATE + target_state.emit( + Event( + EventType.INFERENCE_RESPONSE, + { + "mode": mode, + "model_id": model_id, + "backend": backend, + "response_chars": len(response), + }, + ) + ) + + +APP_STATE = AppState() diff --git a/core/deployment.py b/core/deployment.py new file mode 100644 index 0000000000000000000000000000000000000000..8e4d48059cc0525ee9686de53507143ed7d1e6ce --- /dev/null +++ b/core/deployment.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + +LOCAL_DEPLOYMENT = "local" +SPACE_DEPLOYMENT = "space" +VALID_DEPLOYMENTS = {LOCAL_DEPLOYMENT, SPACE_DEPLOYMENT} +PLACEHOLDER_BACKEND = "placeholder" + + +@dataclass(frozen=True) +class DeploymentPolicy: + mode: str = LOCAL_DEPLOYMENT + + @property + def is_space(self) -> bool: + return self.mode == SPACE_DEPLOYMENT + + @property + def allow_placeholder_backend(self) -> bool: + return not self.is_space + + @property + def allow_demo_mode(self) -> bool: + return not self.is_space + + +def deployment_mode(env_value: str | None = None) -> str: + raw_value = env_value if env_value is not None else os.getenv("WORKBENCH_DEPLOYMENT", "") + value = raw_value.strip().lower() or LOCAL_DEPLOYMENT + if value not in VALID_DEPLOYMENTS: + raise ValueError( + "WORKBENCH_DEPLOYMENT must be 'local' or 'space'; " + f"got {raw_value!r}." + ) + return value + + +def current_policy() -> DeploymentPolicy: + return DeploymentPolicy(deployment_mode()) + + +def filter_backends_for_policy(backends: list[str], policy: DeploymentPolicy) -> list[str]: + if policy.allow_placeholder_backend: + return backends + return [backend for backend in backends if backend != PLACEHOLDER_BACKEND] + + +def default_backend_for_policy( + backends: list[str], + preferred: str, + policy: DeploymentPolicy, +) -> str: + allowed = filter_backends_for_policy(backends, policy) + if preferred in allowed: + return preferred + if allowed: + return allowed[0] + raise ValueError("No real backend is available for this deployment policy.") + + +def ensure_backend_allowed(backend: str, policy: DeploymentPolicy) -> None: + if backend == PLACEHOLDER_BACKEND and not policy.allow_placeholder_backend: + raise ValueError( + "Placeholder inference is disabled in WORKBENCH_DEPLOYMENT=space. " + "Select a real backend such as transformers, Ollama, llama.cpp, " + "SGLang, or an OpenAI-compatible local server." + ) + + +def ensure_demo_mode_allowed(policy: DeploymentPolicy) -> None: + if not policy.allow_demo_mode: + raise ValueError( + "Demo/no-model mode is disabled in WORKBENCH_DEPLOYMENT=space. " + "Use a real OpenBMB model configuration for the deployed app." + ) diff --git a/core/events.py b/core/events.py new file mode 100644 index 0000000000000000000000000000000000000000..65d07cacc9429856c9c30fa6ea00593ff714f8c2 --- /dev/null +++ b/core/events.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class StrEnum(str, Enum): + """Python 3.10-compatible subset of enum.StrEnum.""" + + def __str__(self) -> str: + return str(self.value) + + +class EventType(StrEnum): + """Event names shared across future services.""" + + DATASET_LOADED = "dataset_loaded" + INFERENCE_REQUEST = "inference_request" + INFERENCE_RESPONSE = "inference_response" + UI_ERROR = "ui_error" + FIELD_NOTE_SAVED = "field_note_saved" + TRAINING_STARTED = "training_started" + EXPORT_STARTED = "export_started" + + +@dataclass +class Event: + """A lightweight event payload.""" + + type: EventType + payload: dict[str, Any] = field(default_factory=dict) + + +class EventBus: + """In-process event dispatcher for low-volume app events.""" + + def __init__(self) -> None: + self._handlers: dict[EventType, list[Callable[[Event], None]]] = {} + + def on(self, event_type: EventType) -> Callable: + def decorator(fn: Callable[[Event], None]) -> Callable[[Event], None]: + self._handlers.setdefault(event_type, []).append(fn) + return fn + + return decorator + + def emit(self, event: Event) -> None: + for handler in self._handlers.get(event.type, []): + handler(event) + + +bus = EventBus() diff --git a/core/file_exports.py b/core/file_exports.py new file mode 100644 index 0000000000000000000000000000000000000000..2f1147c36a35a035093beb77f05873efc84a6938 --- /dev/null +++ b/core/file_exports.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from pathlib import Path + + +def copy_text_file_or_empty(source_path: str | Path, output_path: str | Path) -> Path: + """Copy a text file to an export path, or create an empty export.""" + + source = Path(source_path) + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + if source.exists(): + output.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + else: + output.write_text("", encoding="utf-8") + return output diff --git a/core/registry.py b/core/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..e844f2e6fa42a5faeacff2c57cbb2c91b79dc1f2 --- /dev/null +++ b/core/registry.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Registry(Generic[T]): + """Small name-to-service registry.""" + + def __init__(self) -> None: + self._items: dict[str, T] = {} + + def register(self, name: str, item: T) -> None: + self._items[name] = item + + def get(self, name: str) -> T: + if name not in self._items: + available = ", ".join(self._items) + raise KeyError(f"{name!r} is not registered. Available: {available}") + return self._items[name] + + def list(self) -> list[str]: + return list(self._items) diff --git a/core/spaces_runtime.py b/core/spaces_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..bda3adc78881bb33c68db165a89cc9b7fe87bf63 --- /dev/null +++ b/core/spaces_runtime.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from collections.abc import Callable +from importlib import import_module +from types import SimpleNamespace +from typing import Any, TypeVar + +F = TypeVar("F", bound=Callable[..., Any]) +spaces: Any + +try: + spaces = import_module("spaces") +except ImportError: + + def _gpu(*_args: Any, **_kwargs: Any) -> Callable[[F], F]: + def decorator(func: F) -> F: + return func + + return decorator + + spaces = SimpleNamespace(GPU=_gpu) diff --git a/core/tab_feedback.py b/core/tab_feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..9964dfe63003db9f9bca7d9484a0703877f37863 --- /dev/null +++ b/core/tab_feedback.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import Any + +from core.app_state import APP_STATE, AppState +from core.events import Event, EventType + + +def status_ok(message: str) -> str: + return f"Status: {message}" + + +def emit_tab_error( + tab: str, + message: str, + context: dict[str, Any] | None = None, + state: AppState | None = None, +) -> str: + target_state = state or APP_STATE + payload = {"tab": tab, "message": message} + if context: + payload.update(context) + target_state.emit(Event(EventType.UI_ERROR, payload)) + return f"Error: {message}" diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d4dca6fad14f9875f6c29d7fc60b77dac3bbaf9 --- /dev/null +++ b/datasets/__init__.py @@ -0,0 +1 @@ +"""Dataset and correction-loop helpers.""" diff --git a/datasets/field_notes.py b/datasets/field_notes.py new file mode 100644 index 0000000000000000000000000000000000000000..3af4add50834b8704b0badc37b38146be39ff9f7 --- /dev/null +++ b/datasets/field_notes.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import csv +import datetime as dt +import json +import sqlite3 +from contextlib import closing +from dataclasses import asdict, dataclass +from pathlib import Path + +UTC = getattr(dt, "UTC", dt.timezone.utc) # noqa: UP017 + + +@dataclass +class FieldNote: + """One human correction record.""" + + created_at: str + model_id: str + prompt: str + response: str + correction: str + tags: str + image_path: str = "" + video_path: str = "" + use_for_training: bool = True + + @classmethod + def create( + cls, + model_id: str, + prompt: str, + response: str, + correction: str, + tags: str, + image_path: str = "", + video_path: str = "", + use_for_training: bool = True, + ) -> FieldNote: + return cls( + created_at=dt.datetime.now(UTC).isoformat(), + model_id=model_id, + prompt=prompt, + response=response, + correction=correction, + tags=tags, + image_path=image_path, + video_path=video_path, + use_for_training=use_for_training, + ) + + @classmethod + def from_row(cls, row: dict[str, str]) -> FieldNote: + use_for_training = str(row.get("use_for_training", "true")).lower() in { + "1", + "true", + "yes", + } + return cls( + created_at=row["created_at"], + model_id=row["model_id"], + prompt=row["prompt"], + response=row["response"], + correction=row["correction"], + tags=row["tags"], + image_path=row.get("image_path", ""), + video_path=row.get("video_path", ""), + use_for_training=use_for_training, + ) + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +class FieldNoteStore: + """CSV-backed field note storage.""" + + def __init__(self, path: str | Path = "data/field_notes.csv") -> None: + self.path = Path(path) + + def save(self, note: FieldNote) -> Path: + self.path.parent.mkdir(parents=True, exist_ok=True) + is_new = not self.path.exists() + + with self.path.open("a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(note.to_dict())) + if is_new: + writer.writeheader() + writer.writerow(note.to_dict()) + + return self.path + + def list_notes( + self, + corrected_only: bool = False, + tag: str = "", + training_only: bool = False, + ) -> list[FieldNote]: + if not self.path.exists(): + return [] + + with self.path.open(newline="", encoding="utf-8") as f: + rows = list(csv.DictReader(f)) + + notes = [FieldNote.from_row(row) for row in rows] + if corrected_only: + notes = [note for note in notes if note.correction.strip()] + if tag: + notes = [note for note in notes if tag in _split_tags(note.tags)] + if training_only: + notes = [note for note in notes if note.use_for_training] + return notes + + def export_jsonl( + self, + output_path: str | Path = "data/field_notes.jsonl", + corrected_only: bool = True, + training_only: bool = True, + ) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + + notes = self.list_notes( + corrected_only=corrected_only, + training_only=training_only, + ) + with output.open("w", encoding="utf-8") as f: + for note in notes: + f.write(json.dumps(note.to_dict(), ensure_ascii=False) + "\n") + + return output + + def export_hf_dataset( + self, + output_dir: str | Path = "data/hf_field_notes", + corrected_only: bool = True, + training_only: bool = True, + ) -> Path: + target = Path(output_dir) + target.mkdir(parents=True, exist_ok=True) + data_file = self.export_jsonl( + target / "data.jsonl", + corrected_only=corrected_only, + training_only=training_only, + ) + (target / "README.md").write_text( + "# Field Notes Dataset\n\n" + "Local export generated by OpenBMB Local AI Workbench.\n\n" + f"- Data file: `{data_file.name}`\n" + "- Intended split: `train`\n", + encoding="utf-8", + ) + return target + + +class SQLiteFieldNoteStore: + """SQLite-backed field note storage for larger correction loops.""" + + def __init__(self, path: str | Path = "data/field_notes.sqlite") -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._init_schema() + + def _connect(self) -> sqlite3.Connection: + return sqlite3.connect(self.path) + + def _init_schema(self) -> None: + with closing(self._connect()) as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS field_notes ( + created_at TEXT NOT NULL, + model_id TEXT NOT NULL, + prompt TEXT NOT NULL, + response TEXT NOT NULL, + correction TEXT NOT NULL, + tags TEXT NOT NULL, + image_path TEXT NOT NULL, + video_path TEXT NOT NULL, + use_for_training INTEGER NOT NULL + ) + """ + ) + conn.commit() + + def save(self, note: FieldNote) -> Path: + with closing(self._connect()) as conn: + conn.execute( + """ + INSERT INTO field_notes ( + created_at, model_id, prompt, response, correction, tags, + image_path, video_path, use_for_training + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + note.created_at, + note.model_id, + note.prompt, + note.response, + note.correction, + note.tags, + note.image_path, + note.video_path, + int(note.use_for_training), + ), + ) + conn.commit() + return self.path + + def list_notes( + self, + corrected_only: bool = False, + tag: str = "", + training_only: bool = False, + ) -> list[FieldNote]: + with closing(self._connect()) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute( + """ + SELECT created_at, model_id, prompt, response, correction, tags, + image_path, video_path, use_for_training + FROM field_notes + ORDER BY created_at + """ + ).fetchall() + + notes = [ + FieldNote( + created_at=str(row["created_at"]), + model_id=str(row["model_id"]), + prompt=str(row["prompt"]), + response=str(row["response"]), + correction=str(row["correction"]), + tags=str(row["tags"]), + image_path=str(row["image_path"]), + video_path=str(row["video_path"]), + use_for_training=bool(row["use_for_training"]), + ) + for row in rows + ] + if corrected_only: + notes = [note for note in notes if note.correction.strip()] + if tag: + notes = [note for note in notes if tag in _split_tags(note.tags)] + if training_only: + notes = [note for note in notes if note.use_for_training] + return notes + + +def _split_tags(tags: str) -> set[str]: + return {tag.strip() for tag in tags.split(",") if tag.strip()} diff --git a/datasets/loader.py b/datasets/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..a7365717adda509920c4a32495c81be9e5cd342f --- /dev/null +++ b/datasets/loader.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import csv +import json +from dataclasses import dataclass +from importlib import import_module +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class DatasetPreview: + """Small local dataset preview result.""" + + source: str + rows: int + columns: list[str] + samples: list[dict[str, Any]] + + def as_table(self) -> list[list[str]]: + table = [ + ["source", self.source], + ["rows", str(self.rows)], + ["columns", ", ".join(self.columns)], + ] + for index, sample in enumerate(self.samples, start=1): + table.append([f"sample_{index}", json.dumps(sample, ensure_ascii=False)]) + return table + + +@dataclass(frozen=True) +class DatasetStats: + """Basic dataset statistics for local preview and tools.""" + + source: str + rows: int + columns: int + column_names: list[str] + non_empty_by_column: dict[str, int] + + def as_dict(self) -> dict[str, Any]: + return { + "source": self.source, + "rows": self.rows, + "columns": self.columns, + "column_names": self.column_names, + "non_empty_by_column": self.non_empty_by_column, + } + + +@dataclass(frozen=True) +class HuggingFaceDatasetPreview: + """Small preview result for optional Hugging Face datasets integration.""" + + dataset_id: str + split: str + rows: int + columns: list[str] + samples: list[dict[str, Any]] + status: str = "loaded" + + def as_table(self) -> list[list[str]]: + table = [ + ["dataset_id", self.dataset_id], + ["split", self.split], + ["rows", str(self.rows)], + ["columns", ", ".join(self.columns)], + ["status", self.status], + ] + for index, sample in enumerate(self.samples, start=1): + table.append([f"sample_{index}", json.dumps(sample, ensure_ascii=False)]) + return table + + +def preview_local_dataset(path: str | Path, limit: int = 5) -> DatasetPreview: + dataset_path = Path(path) + if not dataset_path.exists(): + raise FileNotFoundError(f"Dataset path not found: {dataset_path}") + if dataset_path.suffix.lower() == ".csv": + return _preview_csv(dataset_path, limit) + if dataset_path.suffix.lower() in {".jsonl", ".ndjson"}: + return _preview_jsonl(dataset_path, limit) + raise ValueError(f"Unsupported dataset format: {dataset_path.suffix}") + + +def dataset_statistics(path: str | Path) -> DatasetStats: + preview = preview_local_dataset(path, limit=0) + non_empty_by_column = {column: 0 for column in preview.columns} + + dataset_path = Path(path) + if dataset_path.suffix.lower() == ".csv": + with dataset_path.open(newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + for column in preview.columns: + if str(row.get(column, "")).strip(): + non_empty_by_column[column] += 1 + else: + with dataset_path.open(encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + row = json.loads(line) + for column in preview.columns: + if str(row.get(column, "")).strip(): + non_empty_by_column[column] += 1 + + return DatasetStats( + source=preview.source, + rows=preview.rows, + columns=len(preview.columns), + column_names=preview.columns, + non_empty_by_column=non_empty_by_column, + ) + + +def preview_huggingface_dataset( + dataset_id: str, + split: str = "train", + limit: int = 5, +) -> HuggingFaceDatasetPreview: + datasets_module = import_module("datasets") + load_dataset = getattr(datasets_module, "load_dataset", None) + if load_dataset is None: + return HuggingFaceDatasetPreview( + dataset_id=dataset_id, + split=split, + rows=0, + columns=[], + samples=[], + status="Hugging Face package 'datasets' is not installed.", + ) + + dataset = load_dataset(dataset_id, split=split) + rows = len(dataset) + columns = list(dataset.column_names) + samples = [dict(dataset[index]) for index in range(min(limit, rows))] + return HuggingFaceDatasetPreview(dataset_id, split, rows, columns, samples) + + +def _preview_csv(path: Path, limit: int) -> DatasetPreview: + samples: list[dict[str, Any]] = [] + rows = 0 + columns: list[str] = [] + + with path.open(newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + columns = list(reader.fieldnames or []) + for row in reader: + rows += 1 + if len(samples) < limit: + samples.append(dict(row)) + + return DatasetPreview(str(path), rows, columns, samples) + + +def _preview_jsonl(path: Path, limit: int) -> DatasetPreview: + samples: list[dict[str, Any]] = [] + rows = 0 + columns: set[str] = set() + + with path.open(encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + rows += 1 + item = json.loads(line) + if not isinstance(item, dict): + raise ValueError("JSONL rows must be objects") + columns.update(item) + if len(samples) < limit: + samples.append(item) + + return DatasetPreview(str(path), rows, sorted(columns), samples) diff --git a/datasets/ocr.py b/datasets/ocr.py new file mode 100644 index 0000000000000000000000000000000000000000..595f2fad9343d26c847b7bb25373cf9ad8506a5b --- /dev/null +++ b/datasets/ocr.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import csv +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from datasets.field_notes import FieldNote, FieldNoteStore + + +@dataclass(frozen=True) +class OCRPrediction: + """One local OCR prediction that may need human correction.""" + + source_path: str + text: str + confidence: float + page: str = "" + + @classmethod + def from_row(cls, row: dict[str, Any]) -> OCRPrediction: + source_path = _first_present(row, ["source_path", "image_path", "file_path", "path"]) + text = _first_present(row, ["text", "prediction", "ocr_text", "response"]) + confidence_raw = _first_present(row, ["confidence", "score", "probability"], "0") + return cls( + source_path=source_path, + text=text, + confidence=_parse_confidence(confidence_raw), + page=str(row.get("page", "")), + ) + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +def load_ocr_predictions(path: str | Path) -> list[OCRPrediction]: + """Load OCR predictions from local CSV, JSONL, or NDJSON files.""" + + source = Path(path) + if not source.exists(): + raise FileNotFoundError(f"OCR prediction file not found: {source}") + + suffix = source.suffix.lower() + if suffix == ".csv": + with source.open(newline="", encoding="utf-8") as f: + return [OCRPrediction.from_row(row) for row in csv.DictReader(f)] + if suffix in {".jsonl", ".ndjson"}: + return [ + OCRPrediction.from_row(json.loads(line)) + for line in source.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + raise ValueError("OCR predictions must be a .csv, .jsonl, or .ndjson file.") + + +def uncertain_predictions( + predictions: list[OCRPrediction], + confidence_threshold: float = 0.8, +) -> list[OCRPrediction]: + return [ + prediction + for prediction in predictions + if prediction.confidence <= confidence_threshold or not prediction.text.strip() + ] + + +def import_uncertain_predictions( + store: FieldNoteStore, + predictions: list[OCRPrediction], + model_id: str, + confidence_threshold: float = 0.8, + tags: str = "ocr,uncertain", +) -> int: + imported = 0 + for prediction in uncertain_predictions(predictions, confidence_threshold): + page_note = f" page {prediction.page}" if prediction.page else "" + note = FieldNote.create( + model_id=model_id, + prompt=f"Review OCR text for {prediction.source_path}{page_note}.", + response=prediction.text, + correction="", + tags=tags, + image_path=prediction.source_path, + use_for_training=False, + ) + store.save(note) + imported += 1 + return imported + + +def export_corrected_ocr_notes( + store: FieldNoteStore, + output_path: str | Path = "data/ocr_corrections.jsonl", +) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + notes = store.list_notes(corrected_only=True, tag="ocr") + with output.open("w", encoding="utf-8") as f: + for note in notes: + row = { + "source_path": note.image_path, + "predicted_text": note.response, + "corrected_text": note.correction, + "model_id": note.model_id, + "created_at": note.created_at, + "tags": note.tags, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + return output + + +def ocr_import_summary(path: str | Path, confidence_threshold: float = 0.8) -> dict[str, object]: + predictions = load_ocr_predictions(path) + uncertain = uncertain_predictions(predictions, confidence_threshold) + return { + "source": str(path), + "rows": len(predictions), + "uncertain_rows": len(uncertain), + "confidence_threshold": confidence_threshold, + "sample": [prediction.to_dict() for prediction in uncertain[:5]], + } + + +def _first_present(row: dict[str, Any], names: list[str], default: str = "") -> str: + for name in names: + value = row.get(name) + if value is not None: + return str(value) + return default + + +def _parse_confidence(value: str) -> float: + try: + return float(value) + except ValueError: + return 0.0 diff --git a/datasets/synthetic.py b/datasets/synthetic.py new file mode 100644 index 0000000000000000000000000000000000000000..c52c81ef0abcbd849d37668cbee82a9101a12422 --- /dev/null +++ b/datasets/synthetic.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class SyntheticExample: + prompt: str + response: str + correction: str + tags: list[str] + source: str = "synthetic" + + def as_dict(self) -> dict: + return asdict(self) + + +def generate_synthetic_examples( + topic: str, + count: int, + tag: str = "synthetic", +) -> list[SyntheticExample]: + if count < 0: + raise ValueError("count must be non-negative") + cleaned_topic = topic.strip() or "local AI workflow" + return [ + SyntheticExample( + prompt=f"Write a concise field note about {cleaned_topic} #{index}.", + response=f"Draft note {index}: {cleaned_topic}.", + correction=f"Corrected note {index}: {cleaned_topic} with concrete evidence.", + tags=[tag, cleaned_topic.replace(" ", "-").lower()], + ) + for index in range(1, count + 1) + ] + + +def validate_synthetic_example(example: SyntheticExample) -> list[str]: + errors = [] + if not example.prompt.strip(): + errors.append("prompt is required") + if not example.response.strip(): + errors.append("response is required") + if not example.correction.strip(): + errors.append("correction is required") + if not example.tags: + errors.append("at least one tag is required") + return errors + + +def quality_filter_examples(examples: list[SyntheticExample]) -> list[SyntheticExample]: + return [ + example + for example in examples + if not validate_synthetic_example(example) + and example.prompt != example.response + and example.response != example.correction + ] + + +def augment_examples( + examples: list[SyntheticExample], + extra_tag: str = "augmented", +) -> list[SyntheticExample]: + augmented = list(examples) + for example in examples: + tags = list(dict.fromkeys([*example.tags, extra_tag])) + augmented.append( + SyntheticExample( + prompt=f"{example.prompt} Include one verification detail.", + response=example.response, + correction=f"{example.correction} Verification detail recorded.", + tags=tags, + source=example.source, + ) + ) + return augmented + + +def export_synthetic_jsonl( + examples: list[SyntheticExample], + output_path: str | Path = "data/synthetic_examples.jsonl", +) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8") as f: + for example in examples: + f.write(json.dumps(example.as_dict(), ensure_ascii=False) + "\n") + return output diff --git a/deployment/__init__.py b/deployment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deployment/hf_space.py b/deployment/hf_space.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5339e3f6df5b160a4bf499174cc5ca0cbdefcc --- /dev/null +++ b/deployment/hf_space.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path + +SPACE_REQUIRED_FILES = [ + "app.py", + "plant_space_app.py", + "requirements.txt", + "README.md", + "config/models.yaml", + "config/training.yaml", +] + + +@dataclass(frozen=True) +class SpaceBuildStatus: + required_files_present: bool + missing_files: list[str] + readme_has_space_metadata: bool + has_space_remote: bool + + def as_dict(self) -> dict: + return asdict(self) + + +def missing_required_files(root: str | Path = ".") -> list[str]: + repo_root = Path(root) + return [path for path in SPACE_REQUIRED_FILES if not (repo_root / path).exists()] + + +def readme_has_space_metadata(root: str | Path = ".") -> bool: + readme_path = Path(root) / "README.md" + if not readme_path.exists(): + return False + text = readme_path.read_text(encoding="utf-8") + return text.startswith("---\n") and "\nsdk: gradio\n" in text and "\napp_file: app.py\n" in text + + +def has_space_remote(remote_output: str, remote_name: str = "space") -> bool: + return any(line.startswith(f"{remote_name}\t") for line in remote_output.splitlines()) + + +def space_build_status(root: str | Path = ".", remote_output: str = "") -> SpaceBuildStatus: + missing = missing_required_files(root) + return SpaceBuildStatus( + required_files_present=not missing, + missing_files=missing, + readme_has_space_metadata=readme_has_space_metadata(root), + has_space_remote=( + has_space_remote(remote_output, "space") + or has_space_remote(remote_output, "space-workbench") + or has_space_remote(remote_output, "space-plant") + ), + ) + + +def create_space_commands( + hf_user: str, + space_name: str = "openbmb-local-ai-workbench", + branch: str = "main", + remote_name: str = "space", + app_file: str = "app.py", +) -> list[str]: + if not hf_user.strip(): + raise ValueError("hf_user is required") + if not space_name.strip(): + raise ValueError("space_name is required") + if not remote_name.strip(): + raise ValueError("remote_name is required") + if not app_file.strip(): + raise ValueError("app_file is required") + return [ + "hf auth login", + f"hf repo create {hf_user}/{space_name} --type space --space-sdk gradio", + f"git remote add {remote_name} https://huggingface.co/spaces/{hf_user}/{space_name}", + f"git push {remote_name} {branch}", + f"Space metadata app_file must be {app_file} for this deployment.", + ] + + +def hackathon_space_commands(branch: str = "main") -> dict[str, list[str]]: + return { + "workbench": create_space_commands( + "build-small-hackathon", + "workbench", + branch, + remote_name="space-workbench", + app_file="app.py", + ), + "plant": create_space_commands( + "build-small-hackathon", + "plant_identification_tool", + branch, + remote_name="space-plant", + app_file="plant_space_app.py", + ), + } diff --git a/docs/ACCEPTANCE_CRITERIA.md b/docs/ACCEPTANCE_CRITERIA.md new file mode 100644 index 0000000000000000000000000000000000000000..3cb40fb915d360bf25838894450b6d54e0ef8c08 --- /dev/null +++ b/docs/ACCEPTANCE_CRITERIA.md @@ -0,0 +1,96 @@ +# Acceptance Criteria + +This file defines how we decide whether something is really done. + +## Global Rule + +Do not mark a feature done unless: + +1. The files exist. +2. The feature is linked from docs where relevant. +3. The behavior was tested locally, or the blocker is written in `IMPLEMENTATION_STATUS.md`. +4. The checklist in `TASKS.md` is updated. +5. Unit and user-story tests are added or updated. + +If a bug or failed check is discovered, add or update a test that reproduces it before marking +the fix done. + +## MVP App + +Done when: + +- `python app.py` launches without errors. +- The Gradio UI opens locally. +- Chat tab returns a response. +- Vision tab accepts an image and returns a response. +- Field Notes tab saves a row to `data/field_notes.csv`. +- README explains that current inference is placeholder/local-safe. +- Unit and user-story tests pass. + +## Testing And Quality + +Done when: + +- Unit tests cover new logic. +- User-story tests cover new workflows. +- `scripts/run_tests.ps1` passes. +- `ruff`, `mypy`, `pylint`, `bandit`, and `pip-audit` are installed or blockers are documented. +- `scripts/run_quality.ps1` passes before release or documents exact blockers. + +## Real Local Inference + +Done when: + +- A user can choose a backend. +- Model loading happens only after an explicit click. +- The app does not download weights on startup. +- At least one small model returns a real response. +- Setup steps are documented. + +## Hugging Face Space + +Done when: + +- Space exists. +- Repo is pushed. +- Build succeeds. +- Public/private Space URL is added to README. +- Any hardware or model limitations are documented. + +## GitHub + +Done when: + +- GitHub repo exists. +- Initial commit is pushed. +- README includes GitHub URL. +- Large files and secrets are excluded. + +## Training + +Done when: + +- Training input format is documented. +- A dry-run or actual LoRA run completes. +- Output checkpoint path is shown. +- Metrics or logs are captured. +- Hardware assumptions are documented. + +## Field Notes + +Done when: + +- Corrections save reliably. +- Corrected records can be exported. +- Export schema is documented. +- Export can feed the training pipeline. + +## Hackathon Submission + +Done when: + +- Space URL is live. +- Demo video is recorded. +- Social post is published or drafted. +- Submission form fields are complete. +- Model-size compliance is documented. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000000000000000000000000000000000000..702d15273d2d317215967a057a1b12e39f2d53df --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,575 @@ +# Architecture + +The project is intentionally small at first. The PRD describes a large workbench; this repo starts +with the smallest version that can grow into it. + +## High-Level Flow + +```text +app.py + loads config/models.yaml + configures lightweight logging + builds Gradio tabs + passes model catalog to UI modules + +ui/* + defines each Gradio tab + calls service classes + emits local app events for inference, datasets, and field notes + uses shared progress settings for callback loading indicators + +agent/* + holds deterministic local agent planning and trace export helpers + +models/* + holds model catalog, local backend config, and inference services + +datasets/* + stores dataset, synthetic data, and correction-loop helpers + +mcp_tools/* + holds local tool functions, VINDEX call planning, and Gradio-native MCP bridge metadata + +config/* + holds model and training settings + +training/* + holds non-executing training, LoRA request, evaluation, and export planning helpers + +tracking/* + holds local JSONL tracing and optional Trackio integration + +deployment/* + holds Hugging Face Space deployment planning and validation helpers + +plant/* + holds the first reference domain app built from the template + can run standalone with python -m plant.app --no-model + keeps heavy model dependencies optional + +core/* + shared app state, event, logging, and registry helpers +``` + +## Files And Classes + +### `app.py` + +Builds and launches the Gradio app. + +- `build_app()` creates the Gradio `Blocks` app. +- Loads the model catalog from `config/models.yaml`. +- Registers the current UI tabs. +- `APP_CSS` defines compact responsive layout rules for app width, mobile padding, scrollable tabs, + and button touch targets. + +### `plant/app.py` + +Standalone Plant Discovery reference app built from the template. + +- `build_app(no_model=True)` creates a Gradio app without loading model weights. +- Loads `plant/models.yaml`. +- Builds a local species index. +- Reuses `datasets.field_notes.FieldNoteStore` for corrections. +- Uses `DemoPlantVisionService` for screenshots/tests or `PlantVisionService` for OpenBMB + MiniCPM-V zero-shot and fine-tuned adapter inference. + +### `plant/plant_service.py` + +Domain service and schema for Plant Discovery. + +- `PlantID` is the structured output schema. +- `DemoPlantVisionService` provides deterministic no-model results. +- `PlantVisionService` lazy-loads optional MiniCPM-V dependencies only during identification. +- `PlantVisionService.from_config(..., "plant_vlm_finetuned")` can load a PEFT adapter after a real + adapter repo is configured. +- `extract_json_object()` and `parse_plant_response()` make model JSON output testable. + +### `plant/training.py` + +Non-executing training planner for Plant Discovery. + +- `build_plant_training_plan()` returns SWIFT and LLaMA-Factory command previews. +- `plant_training_dependency_report()` reports optional training dependency availability. +- `write_llamafactory_dataset_info()` writes a dataset-info preview for LLaMA-Factory workflows. +- Training is never started by the Gradio UI or script. + +### `plant/plant_loader.py` + +Domain data and export helpers for Plant Discovery. + +- `PlantRecord` normalizes plant examples into training rows. +- `LocalFolderLoader` maps species folders to image metadata. +- `SpeciesIndexBuilder` builds a no-network species index with demo fallback. +- `FieldNotesPlantExporter` exports corrected field notes to plant training JSONL. + +### `plant/plant_tab.py` + +Focused Gradio UI for Plant Discovery. + +- Identify tab uploads images and renders a safe escaped result card. +- Field Guide tab searches the species index. +- Corrections tab saves and exports training-ready corrections. +- Stats tab summarizes species and correction counts. +- Training is represented as a non-executing plan, not a subprocess. + +### `plant/plant_tools.py` + +Optional local/MCP tools for Plant Discovery. + +- Pure functions can be tested without an MCP server. +- `build_mcp_server()` imports `mcp` only when explicitly requested. +- Tools expose identify, species search, correction save/export, stats, and training plan. + +### `models/model_catalog.py` + +Reads model configuration and turns it into typed Python objects. + +- `ModelInfo` describes one configured model. +- `load_model_catalog(path)` reads YAML and returns all configured models. +- `model_choices(catalog, model_type)` filters models for a UI dropdown. +- `model_summary(model)` returns display metadata for the Gradio JSON panel. +- `backend_capabilities` maps each model to supported local backend capabilities. + +### `models/placeholder_service.py` + +Deterministic placeholder model service used before real inference is wired. + +- `PlaceholderModelService.chat()` returns a deterministic text response. +- `PlaceholderModelService.vision_chat()` returns a deterministic image/prompt response. + +This file should be replaced or complemented by real services such as: + +- `ollama_service.py` +- `llama_cpp_service.py` +- `openai_compatible_service.py` +- `sglang_runner.py` +- `minicpm_vision.py` +- `transformers_text.py` +- `sglang_service.py` + +### `models/base.py` + +Defines service contracts and backend status records. + +- `BackendStatus` describes whether a backend is available. +- `TextModelService` is the text chat protocol. +- `VisionModelService` is the vision chat protocol. + +### `models/ollama_service.py` + +Ollama-backed local inference client. + +- Checks whether `ollama` is installed and reachable. +- Sends text and vision chat requests to `http://127.0.0.1:11434/api/chat`. +- Lists locally available Ollama models through `/api/tags`. +- Builds explicit `ollama pull ` commands for the Status tab. +- Does not pull or download models automatically. + +### `models/llama_cpp_service.py` + +llama.cpp HTTP client for local GGUF inference. + +- Checks whether `llama-server` is installed and reachable. +- Builds explicit `llama-server -m ` commands. +- Supports `--mmproj ` command metadata for multimodal models. +- Sends text chat requests to `/v1/chat/completions`. +- Does not download GGUF files or start background servers automatically. + +### `models/local_backend_config.py` + +User-local backend settings stored under ignored `data/local_backends.yaml`. + +- `LocalBackendConfig` stores llama.cpp server URL, OpenAI-compatible base URL, optional served + model name, GGUF path, mmproj path, context length, and GPU layers. +- `save_local_backend_config()` writes local-only settings without touching tracked model config. +- `build_llama_server_command()` returns the explicit command the user can run. +- `local_backend_summary()` reports file status and confirms no startup downloads or automatic model loads. + +### `models/openai_compatible_service.py` + +Local OpenAI-compatible chat client for LM Studio, vLLM-style servers, or similar local endpoints. + +- Checks `/v1/models` for reachability. +- Sends text chat requests to `/v1/chat/completions`. +- Supports an optional served-model-name override for tools such as LM Studio. +- Returns visible unavailable/request-failed messages instead of crashing the Gradio callback. +- Does not call cloud APIs or download model weights. + +### `models/llama_cpp_python_service.py` + +Optional direct Python binding backend for GGUF inference. + +- Checks whether `llama_cpp` is importable. +- Requires an explicit local GGUF path. +- Does not download model files. +- Provides text chat through `Llama.create_chat_completion()`. +- Vision support remains routed through llama-server until mmproj/image serialization is wired. + +### `models/minicpm_vision.py` + +Optional MiniCPM vision backend. + +- Checks whether the `transformers` package is available. +- Lazy-loads `AutoProcessor` and `AutoModelForImageTextToText` only when selected. +- Formats image/text messages for image-text-to-text generation. +- Maps thinking mode into the prompt template. +- Provides a video support plan for future local frame sampling. + +### `models/sglang_runner.py` + +SGLang local server planner and OpenAI-compatible chat client. + +- Builds an explicit `python -m sglang.launch_server` command. +- Includes MiniCPM tool parser configuration. +- Checks `/health`, sends chat requests to `/v1/chat/completions`, and can request `/shutdown`. +- Does not install SGLang, start a process, download model weights, or load a model on app startup. + +### `models/vllm_runner.py` + +vLLM local server planner and OpenAI-compatible chat client. + +- Builds explicit `vllm serve ` command plans. +- Checks `/health`, parses Prometheus-style `/metrics`, and sends chat requests to + `/v1/chat/completions`. +- Logs parsed benchmark metrics through `TrackingClient`. +- Does not install vLLM, start a process, download model weights, or load a model on app startup. + +### `models/transformers_text.py` + +Optional Transformers text backend. + +- Checks whether the `transformers` package is installed. +- Lazy-loads `AutoTokenizer` and `AutoModelForCausalLM` only when the backend is selected. +- Reads `trust_remote_code`, device map, dtype, max token, and temperature settings from explicit config. +- Provides a simple token-list streaming helper for future Gradio streaming wiring. +- Does not download model weights on startup. + +### `models/service_factory.py` + +Creates the selected backend service for the UI. + +- `TEXT_SERVICE_REGISTRY` registers available text backend factories. +- `VISION_SERVICE_REGISTRY` registers available vision backend factories. +- `create_text_service()` chooses placeholder, llama.cpp, llama-cpp-python, Ollama, + OpenAI-compatible, SGLang, or Transformers text service. +- `create_vision_service()` chooses placeholder, llama.cpp, llama-cpp-python, Ollama, or + Transformers MiniCPM vision service. +- `backend_statuses()` reports current backend availability. +- llama.cpp, llama-cpp-python, and OpenAI-compatible services read ignored local backend settings + when selected. + +### `ui/chat_tab.py` + +Builds the text chat tab. + +- Shows text models from the catalog. +- Displays selected model metadata. +- Calls the selected backend service. +- Emits inference request and response events. + +### `ui/vision_tab.py` + +Builds the vision tab. + +- Shows vision models from the catalog. +- Accepts an image and prompt. +- Calls the selected backend service. +- Emits inference request and response events. + +### `ui/dataset_tab.py` + +Local dataset preview surface. + +- Previews local CSV, JSONL, and NDJSON files. +- Previews Hugging Face datasets when the optional external `datasets` package is installed. +- Shows source, row count, columns, and sample rows. +- Calculates basic local dataset statistics. +- Emits dataset loaded events. + +Future behavior: + +- Serve dataset tools through the selected MCP path. + +### `ui/train_tab.py` + +Training planning and local evaluation surface. + +- Builds a LoRA dry-run training plan without launching training. +- Builds a non-executing LoRA trainer request with dependency status. +- Shows SWIFT/LLaMA-Factory vision fine-tuning plan. +- Shows checkpoint output path, validation status, and hardware notes. +- Runs local base-vs-tuned evaluation from newline-separated response text. +- Shows exact-match summary and a qualitative eval table. +- Logs tuned evaluation reports to `data/eval_results.jsonl`. + +Future behavior: + +- Start LoRA training. +- Show loss and metrics. +- Write Trackio traces. + +### `ui/vllm_tab.py` + +vLLM local serving planner. + +- Builds explicit `vllm serve` command plans. +- Checks local vLLM `/health`. +- Fetches and parses `/metrics`. +- Logs vLLM benchmark metrics through local JSONL/Trackio fallback tracking. +- Does not install vLLM, start a process, download models, or load weights on startup. + +### `ui/export_tab.py` + +GGUF export planning surface. + +- Selects a configured model and quantization. +- Shows official GGUF download command plans when the model has GGUF metadata. +- Shows local HF-to-GGUF conversion and llama.cpp quantization command plans. +- Lists files already present under the selected export directory. +- Exposes existing exported files through a Gradio download output. +- Does not execute downloads, conversion, or quantization. + +Future behavior: + +- Execute downloads and conversions after explicit user action. + +### `ui/notes_tab.py` + +Field notes implementation. + +- Saves prompt, model response, correction, and tags to `data/field_notes.csv`. +- Captures optional image path, video path, and a use-for-training flag. +- Exports corrected notes to JSONL. +- Exports local Hugging Face Dataset-style files under `data/hf_field_notes`. +- Imports uncertain OCR predictions for human correction. +- Exports corrected OCR rows to JSONL. +- Emits field note saved events. + +Future behavior: + +- Push corrected notes to a remote Hugging Face Dataset after login. +- Feed notes into fine-tuning. + +### `ui/traces_tab.py` + +Local trace and tracking preview. + +- Shows manual trace event previews. +- Shows recent local app events. +- Shows JSONL trace rows and tracking status. +- Exports local traces to `exports/traces.jsonl`. +- Calls Trackio only when the optional package is installed and enabled. + +### `ui/agent_tab.py` + +Local non-autonomous agent mode. + +- Drafts a research-plan-implement-verify trace. +- Saves agent traces to `data/agent_traces.jsonl`. +- Exports trace JSONL and local HF Dataset-style trace files. +- Does not execute shell commands, commit, push, deploy, download models, or call external services. + +### `ui/status_tab.py` + +Shows configured models and backend metadata. + +- Helps verify model-size compliance and backend status. +- Provides local llama.cpp settings, GGUF/mmproj file pickers, and command generation. +- Provides LM Studio/OpenAI-compatible base URL, optional model-name storage, and reachability check. +- Provides SGLang command planning, health check, and shutdown request controls. + +### `datasets/field_notes.py` + +Field note data model and CSV store. + +- `FieldNote` captures prompt, response, correction, tags, and timestamp. +- `FieldNote` also captures optional image/video paths and a training inclusion flag. +- `FieldNoteStore.save()` persists notes to `data/field_notes.csv`. +- `FieldNoteStore.list_notes()` filters by correction, tag, and training inclusion. +- `FieldNoteStore.export_jsonl()` writes training-ready JSONL. +- `FieldNoteStore.export_hf_dataset()` writes local HF Dataset-style files. +- `SQLiteFieldNoteStore` stores and lists notes in SQLite for larger correction loops. + +### `datasets/loader.py` + +Dataset preview and statistics helpers. + +- `preview_local_dataset()` previews CSV, JSONL, and NDJSON files. +- `dataset_statistics()` reports row count, column count, names, and non-empty counts. +- `preview_huggingface_dataset()` optionally uses the external Hugging Face `datasets` package. + +### `datasets/synthetic.py` + +Deterministic local synthetic data helpers. + +- `generate_synthetic_examples()` creates local prompt/response/correction examples. +- `validate_synthetic_example()` checks schema requirements. +- `quality_filter_examples()` removes incomplete or low-value examples. +- `augment_examples()` creates deterministic variants for workflow testing. +- `export_synthetic_jsonl()` writes JSONL without external services. + +### `datasets/ocr.py` + +Local OCR correction helpers. + +- `OCRPrediction` stores source path, predicted text, confidence, and optional page. +- `load_ocr_predictions()` loads local `.csv`, `.jsonl`, and `.ndjson` prediction files. +- `uncertain_predictions()` filters rows at or below a confidence threshold or with empty text. +- `import_uncertain_predictions()` creates Field Notes correction tasks for uncertain rows. +- `export_corrected_ocr_notes()` writes corrected OCR examples to JSONL for evaluation or training. +- `ocr_import_summary()` previews uncertain rows for the Field Notes tab. + +### `mcp_tools/tools.py` + +Local MCP-style tools. + +- `dataset_stats_tool()` returns local dataset statistics. +- `hf_dataset_preview_tool()` previews Hugging Face datasets when optional dependencies exist. +- `safe_calculator_tool()` evaluates numeric arithmetic only. +- `model_inference_tool()` routes text prompts through the selected model service. +- `tool_registry()` returns the local tool map for a future MCP endpoint. + +### `mcp_tools/vindex_tool.py` + +Non-executing VINDEX integration boundary. + +- Defines the eight VINDEX PRD methods and their local FastAPI paths. +- `build_vindex_call_plan()` validates method names and builds endpoint/payload plans. +- Caps `star_spread.n_neighbors` at 5 and `calibrated_edit.causal_window` at 3 based on the PRD + safety notes. +- `vindex_dependency_report()` checks whether the optional `vindex` package or local health + endpoint is available. +- `vindex_verification_report()` combines dependency status with a safe call plan and keeps + execution disabled until the local VINDEX install is verified. + +### `mcp_tools/bridge.py` + +Gradio-native MCP bridge metadata and local invocation helper. + +- `MCP_PATH` documents `/gradio_api/mcp/sse`. +- `mcp_manifest()` returns the selected mode, path, and tool definitions. +- `invoke_mcp_tool()` verifies local tool invocation by name. + +### `agent/runner.py` + +Deterministic local agent trace runner. + +- `AGENT_SYSTEM_PROMPT` defines the agent behavior contract. +- `run_agent_loop()` produces research, plan, implement, and verify trace steps. +- `run_paper_to_code_loop()` produces paper-to-code research, plan, implement, and verify trace steps. +- `default_safety_gates()` lists the non-autonomous safety requirements. +- `save_agent_trace()` appends traces to JSONL. +- `export_agent_traces()` exports trace JSONL. +- `export_agent_traces_hf_dataset()` writes local HF Dataset-style trace files. +- The runner can call safe local tools, but it is not autonomous. + +### `core/file_exports.py` + +Shared export helper. + +- `copy_text_file_or_empty()` copies a text artifact to an export path or creates an empty one. + +### `training/export.py` + +Non-executing GGUF export planning. + +- `detect_llama_cpp_tools()` checks `llama-server`, `llama-cli`, and `llama-quantize`. +- `build_export_plan()` creates explicit download, conversion, and quantization command plans. +- `list_exported_files()` lists generated/local export files. +- `ExportPlan.as_dict()` marks that commands are not executed and no startup downloads occur. + +### `training/evaluation.py` + +Local deterministic evaluation helpers. + +- `default_prompt_cases()` returns a small built-in prompt test set. +- `load_prompt_cases()` loads prompt/expected pairs from JSONL. +- `evaluate_responses()` computes exact-match rows and a qualitative table. +- `perplexity_from_losses()` computes perplexity from explicit negative log likelihood values. +- `compare_base_vs_tuned()` reports exact-match delta. +- `log_eval_report()` appends JSONL evaluation results. + +### `training/lora_trainer.py` + +Non-executing LoRA trainer request builder. + +- `lora_dependency_report()` reports PEFT, TRL, Transformers, and Torch availability. +- `build_lora_training_request()` combines the training plan with dependency status and a command + preview. +- `vision_finetuning_plan()` documents SWIFT/LLaMA-Factory as the future MiniCPM-V fine-tuning path. +- Keeps `execute_training` false until dependencies, hardware, and dataset schema are approved. + +### `training/reward_eval.py` + +Deterministic local reward-style evaluation helpers. + +- `RewardEvaluator.evaluate()` scores supplied responses with transparent lexical heuristics. +- `best_of_n()` selects the highest-scoring candidate without model calls. +- `create_dpo_pairs()` creates chosen/rejected pairs for DPO-style datasets. +- `eval_lora_vs_base()` compares base and LoRA response rewards. + +### `training/planner.py` + +Non-executing LoRA training planner. + +- `load_training_config()` reads LoRA and training settings from `config/training.yaml`. +- `build_training_plan()` creates a dry-run plan with checkpoint output path. +- `validate_training_plan()` checks dataset existence and numeric training settings. +- `training_hardware_notes()` documents practical local hardware expectations. + +### `tracking/trackio_client.py` + +Tracking client with JSONL fallback. + +- `load_tracking_config()` reads Trackio settings from `config/training.yaml`. +- `TrackingClient.init()` starts Trackio only when enabled and installed. +- `TrackingClient.log()` always writes local JSONL and optionally forwards to Trackio. +- `TrackingClient.finish()` closes optional Trackio state. +- `export_traces()` copies local traces to `exports/traces.jsonl`. +- `read_trace_rows()` returns recent local trace rows for the UI. + +### `core/events.py` + +Small event bus reserved for future cross-module events. + +- `EventType` names app events. +- `UI_ERROR` records visible tab-level failures. +- `Event` carries event data. +- `EventBus` registers handlers and emits events. + +### `core/app_state.py` + +Shared local app state. + +- `AppState.emit()` records events, logs them, and dispatches them through `EventBus`. +- `AppState.emit()` also writes trace events through `TrackingClient`. +- `AppState.recent_events()` returns local trace previews for the Traces tab. +- `emit_inference_response()` records shared response metadata. + +### `core/tab_feedback.py` + +Formats tab status text and emits `ui_error` events for visible tab-level failures. + +### `ui/progress.py` + +Defines the shared Gradio progress mode used by tab button callbacks. + +### `core/app_logging.py` + +Lightweight logging setup. + +- `configure_app_logging()` configures compact process logging once. + +### `core/registry.py` + +Generic registry helper. + +- `Registry.register(name, item)` stores a service. +- `Registry.get(name)` retrieves a service. +- `Registry.list()` lists registered services. + +## Current Design Rule + +The app must not download model weights on startup. Model loading should happen only after the +user chooses a backend/model and clicks an explicit action. diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md new file mode 100644 index 0000000000000000000000000000000000000000..78a030091c9a4d446dffc88be411168c1773bfbf --- /dev/null +++ b/docs/EXTENDING.md @@ -0,0 +1,125 @@ +# How To Extend The Workbench + +## Add A New Model + +1. Open `config/models.yaml`. +2. Add a new model entry. +3. Set `type` to `text`, `vision`, or `omnimodal`. +4. Keep `parameters_b` at or below 32 for hackathon eligibility. +5. Keep `backend: placeholder` until a real service supports it. + +Example: + +```yaml +models: + my_model: + hf_id: org/model-name + display_name: My Model + type: text + parameters_b: 7 + backend: placeholder + context_length: 32768 + local_first: true + notes: Why this model is useful. +``` + +## Add A Real Backend + +Create a service in `models/`, for example: + +```text +models/ollama_service.py +``` + +The service should expose a small interface: + +```python +class OllamaService: + def chat(self, system_prompt: str, user_prompt: str) -> str: + ... +``` + +Then update the service factory or relevant UI tab to choose between placeholder and real services. + +## Add A New Gradio Tab + +1. Create `ui/new_tab.py`. +2. Add a `build_new_tab(...)` function. +3. Import it in `app.py`. +4. Add it inside the `gr.Tabs()` block. +5. Update `docs/ARCHITECTURE.md`. +6. Add a checklist item in `docs/TASKS.md`. +7. Update `docs/IMPLEMENTATION_STATUS.md`. + +## Add Field Notes Export + +Next useful field notes upgrade: + +- Add a button to export `data/field_notes.csv` to JSONL. +- Add a button to upload that JSONL as a Hugging Face Dataset. +- Document the dataset schema in `README.md`. + +Suggested JSONL schema: + +```json +{"model_id":"minicpm5_1b","prompt":"...","response":"...","correction":"...","tags":["demo"]} +``` + +## Add OCR Corrections + +The local OCR extension starts from prediction files rather than running an OCR engine directly. +Use `.csv`, `.jsonl`, or `.ndjson` rows with fields like: + +```json +{"source_path":"receipt.png","text":"Tota1 12.30","confidence":0.54} +``` + +The Field Notes tab can preview uncertain rows, import them as correction tasks, and export +corrected OCR rows to `data/ocr_corrections.jsonl`. The intended wiring is: + +```text +OCR predictions -> uncertain Field Notes -> corrected JSONL/HF Dataset -> training/evaluation +``` + +## Add VINDEX Execution + +The current VINDEX integration is a safety boundary, not an edit runner. It validates the eight PRD +methods, builds non-executing local FastAPI call plans, and reports whether a local VINDEX package +or `http://127.0.0.1:8765/health` server is available. + +Before allowing execution: + +1. Verify the local VINDEX package or FastAPI server. +2. Re-check the PRD bug list: GPU cache cleanup, dead-code paths, star-spread over-editing, and + causal-window limits. +3. Keep `star_spread.n_neighbors <= 5` and `calibrated_edit.causal_window <= 3` until the scaling + formula is validated. +4. Add protected-relation tests for every edit workflow. +5. Only then add an explicit user-triggered execute button or MCP tool. + +## Add Training + +Training should be added only after local inference works. + +Recommended order: + +1. Export field notes to JSONL. +2. Load JSONL as a dataset. +3. Add PEFT/TRL LoRA for text model. +4. Add Trackio logging. +5. Add checkpoint output folder. +6. Add README instructions. + +## Add Hugging Face Space Deployment + +After the local app runs: + +```powershell +.venv\Scripts\python.exe scripts\plan_hf_space.py --user +huggingface-cli login +huggingface-cli repo create openbmb-local-ai-workbench --type space --space-sdk gradio +git remote add space https://huggingface.co/spaces//openbmb-local-ai-workbench +git push space main +``` + +Never commit Hugging Face tokens. diff --git a/docs/HACKATHON_SUBMISSION.md b/docs/HACKATHON_SUBMISSION.md new file mode 100644 index 0000000000000000000000000000000000000000..3f31e93b412caa5fb94d888412d3e1be502fbc54 --- /dev/null +++ b/docs/HACKATHON_SUBMISSION.md @@ -0,0 +1,103 @@ +# Hackathon Submission Package + +This document collects the judge-facing story, demo flow, and submission assets for the Gradio +Hugging Face Build Small Hackathon. + +## Track + +Recommended track: Backyard AI. + +Reason: the app is local-first, small-model focused, and designed to help a solo builder inspect, +test, correct, and document OpenBMB model workflows on their own machine before moving to a Space. + +## Project Story + +OpenBMB Local AI Workbench is a Gradio app for trying small OpenBMB models locally, capturing +human corrections as field notes, and turning those notes into training or evaluation artifacts. + +## Target User + +The target user is a hackathon builder or small-team AI tinkerer who wants a practical local +workflow before committing to cloud deployment, GPU rentals, or a larger training stack. + +## Measurable Benefit + +The app reduces setup uncertainty by keeping model choices, backend availability, field-note +exports, traces, and deployment next steps visible in one Gradio surface. + +## Final Model Family + +Primary family: OpenBMB MiniCPM. + +Current configured models: + +| Config ID | Model | Parameters | Role | +| --- | --- | ---: | --- | +| `minicpm5_1b` | `openbmb/MiniCPM5-1B` | 1B | local text baseline | +| `minicpm5_1b_thinking` | `openbmb/MiniCPM5-1B-Thinking` | 1B | reasoning/text variant | +| `minicpm41_8b` | `openbmb/MiniCPM4.1-8B` | 8B | long-context text candidate | +| `minicpm_v46` | `openbmb/MiniCPM-V-4.6` | 1.3B | vision candidate | +| `minicpm_v46_thinking` | `openbmb/MiniCPM-V-4.6-Thinking` | 1.3B | vision reasoning candidate | +| `minicpm_o45` | `openbmb/MiniCPM-o-4.5` | 8B | omnimodal stretch candidate | + +All configured models are at or below the 32B hackathon limit. + +## Badge Targets + +- Local-first: yes, through verified llama.cpp CLI, llama-cpp-python GGUF, LM Studio/OpenAI-compatible, and OpenBMB MiniCPM-V Plant image paths; Ollama and SGLang remain setup paths until generation is verified. +- llama.cpp: target badge path; requires local llama.cpp install and GGUF model verification. +- Open trace: yes, through local JSONL tracking and trace export. +- Field notes/report: yes, through corrected field notes, JSONL export, and local HF Dataset-style export. + +## Demo Flow + +1. Open the Gradio app locally. +2. Show the Status tab and explain model-size compliance plus backend availability. +3. Use Chat with the local `llama-cpp-python` GGUF backend to show a visible real response. +4. Use the Dataset tab to preview a local JSONL/CSV training candidate. +5. Save a correction in Field Notes and export corrected rows to JSONL. +6. Open Traces to show local event history and optional Trackio status. +7. Open Export to show GGUF conversion and quantization planning. +8. Explain the remaining llama.cpp mmproj, Ollama, SGLang, and Space build verification tasks. +9. Show the GitHub repo and, when available, the Hugging Face Space URL. + +## Screenshot Assets + +- Workbench home: `assets/e2e/workbench/01-workbench-home.png` +- Workbench backend status: `assets/e2e/workbench/05-backend-status.png` +- Plant tool home: `assets/e2e/plant/01-plant-home.png` +- Plant corrections export: `assets/e2e/plant/03-corrections-export.png` + +## Demo Video Script + +1. "This is OpenBMB Local AI Workbench, a Gradio app for small-model local experimentation." +2. "The model registry keeps every configured model below 32B parameters." +3. "The app starts safely in placeholder mode, so it never downloads model weights on startup." +4. "The Status tab shows which local backends are configured or missing." +5. "A user can try a prompt, capture a correction, and export those corrections as training data." +6. "The Traces tab records local workflow events for reproducibility." +7. "The Export tab prepares explicit GGUF conversion and quantization commands." +8. "The next deployment step is pushing this same Gradio app to a Hugging Face Space." + +## Social Post Draft + +Built OpenBMB Local AI Workbench for the Gradio Hugging Face Build Small Hackathon: a local-first +Gradio app for testing MiniCPM models, collecting field-note corrections, exporting training data, +planning GGUF/llama.cpp workflows, and keeping traceable evidence of small-model experiments. + +GitHub: https://github.com/Ckal/codex +Workbench Space: https://huggingface.co/spaces/build-small-hackathon/workbench +Plant Space: https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool + +## Submission Checklist + +- GitHub URL: https://github.com/Ckal/codex +- Workbench Space URL: https://huggingface.co/spaces/build-small-hackathon/workbench +- Plant Identification Tool Space URL: https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool +- Space build verification: blocked until `hf auth login --force` is run with a fresh token +- Demo video URL: pending +- Social post URL: pending +- Field notes/report URL: pending +- Final track: Backyard AI +- App name: OpenBMB Local AI Workbench +- Deadline: June 15, 2026 diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000000000000000000000000000000000000..29ec4c54b19a8c5dc253149850cc9c809611479b --- /dev/null +++ b/docs/IMPLEMENTATION_STATUS.md @@ -0,0 +1,186 @@ +# Implementation Status + +This file answers: "How do we know what is done?" + +An item is done only when: + +1. The code or document exists. +2. It is linked from the relevant docs. +3. It has been manually reviewed. +4. If it is executable, it has been run locally or the blocker is documented. + +## Current Status + +| Area | Status | Evidence | +| --- | --- | --- | +| PRDs | Done | `HF_PRD_v1.md`, `HF_PRD_ext.md` exist at repo root | +| Codex guidance | Done | `AGENTS.md` exists | +| Docs folder | Done | `docs/` exists with task, usage, architecture, extension docs | +| README | Done | `README.md` exists | +| Hackathon submission package | Drafted | `docs/HACKATHON_SUBMISSION.md` contains track, story, user, benefit, model compliance, demo flow, video script, social post draft, and submission checklist | +| Critical improvement roadmap | Done | `docs/ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md` gives a hard judge-oriented rating, architecture/security critique, and second roadmap | +| Template how-to | Done | `docs/TEMPLATE_HOWTO.md` documents how to build new domain apps from the template | +| Plant Discovery plan | Done | `docs/PLANT_DISCOVERY_APP_PLAN.md` tracks the first reference app and remaining work | +| Plant model/training how-to | Done | `docs/PLANT_MODEL_AND_TRAINING_HOWTO.md` documents demo, OpenBMB zero-shot, fine-tuned adapter mode, correction export, training plan, and adapter configuration | +| Gradio app shell | Implemented and launch-verified | `app.py` builds and foreground launch holds the server open; not currently running | +| Model config | Implemented | `config/models.yaml` exists | +| Expanded model config | Implemented | MiniCPM text, thinking, 4.1, V 4.6, V thinking, omnimodal entries, NVIDIA Nemotron Nano 9B v2, GGUF metadata, and backend capability metadata including OpenAI-compatible text serving | +| Catalog validation | Implemented | `validate_catalog()` and Status tab warnings | +| Training config | Implemented | `config/training.yaml` exists | +| Placeholder chat | Implemented | `ui/chat_tab.py`, `models/placeholder_service.py` | +| Placeholder vision | Implemented | `ui/vision_tab.py`, `models/placeholder_service.py` | +| Service abstraction | Implemented | `models/base.py`, `models/service_factory.py` | +| Local backend config | Implemented | `models/local_backend_config.py` saves ignored local settings in `data/local_backends.yaml` | +| llama.cpp backend | Implemented and locally verified for text CLI | `models/llama_cpp_service.py`; Status tab can pick GGUF/mmproj paths and build a command using `C:\llama-b9587-bin-win-cuda-13.3-x64\llama-server.exe`; `llama-cli.exe` generated a real response from a local GGUF | +| llama-cpp-python backend | Implemented and locally verified | `models/llama_cpp_python_service.py`; `llama_cpp 0.3.8` generated a real local response from `Llama-3.2-1B-Instruct-Q4_K_M.gguf`; Workbench Playwright captures this response | +| Ollama backend | Implemented, not locally verified | `models/ollama_service.py`; Status tab lists local Ollama models and prepares explicit `ollama pull` commands; Ollama executable not found on PATH | +| OpenAI-compatible backend | Implemented and live-verified | `models/openai_compatible_service.py`; Status tab stores LM Studio/vLLM-style base URL and optional served model name, checks `/v1/models`, and posts to `/v1/chat/completions` only when selected; verified `http://192.168.188.37:1234` with `llama-3.2-1b-instruct` | +| Transformers text backend | Implemented, package installed | `models/transformers_text.py`; lazy-loads tokenizer/model only when selected; `transformers 5.10.2` is installed for MiniCPM-V support | +| MiniCPM vision backend | Implemented and Plant-verified | `models/minicpm_vision.py` and `plant/plant_service.py` use `AutoProcessor` and `AutoModelForImageTextToText` lazily; `assets/plant_sample.jpg` produced a structured OpenBMB MiniCPM-V result | +| SGLang backend | Implemented, optional local backend | `models/sglang_runner.py`; builds explicit local start commands, reports health, sends OpenAI-compatible chat requests, and provides a shutdown request; SGLang server launch remains unverified and is not a root Space dependency | +| App state | Implemented | `core/app_state.py` records local events and dispatches through `EventBus`; `core/tab_feedback.py` emits tab-level UI errors | +| Service registry | Implemented | `models/service_factory.py` registers text and vision backend factories | +| Dataset tab | Partial | Local CSV/JSONL preview, optional HF dataset preview, schema, split selector, row count, samples, stats, dataset event emission, and tab-level error status | +| Local dataset preview | Implemented | CSV, JSONL, NDJSON preview and statistics via `datasets/loader.py` | +| MCP tools | Implemented locally, Gradio MCP path selected | `mcp_tools/tools.py` provides dataset stats, HF dataset preview/search-style helper, safe calculator, and model inference tool functions; `mcp_tools/bridge.py` documents `/gradio_api/mcp/sse` and verifies local invocation | +| VINDEX boundary | Implemented locally, execution disabled | `mcp_tools/vindex_tool.py` validates the eight PRD methods, builds non-executing call plans, caps risky edit parameters, and reports local package/server availability | +| Training tab | Partial | `ui/train_tab.py` builds a LoRA dry-run plan, checkpoint output path, hardware notes, local deterministic evaluation, and optional loss-based perplexity summary | +| Training planner | Implemented, non-executing | `training/planner.py` parses LoRA/training config, validates dry runs, and never starts training | +| LoRA trainer planner | Implemented locally, execution disabled | `training/lora_trainer.py` reports PEFT/TRL/Transformers/Torch availability, builds a non-executing LoRA request, and documents SWIFT/LLaMA-Factory vision fine-tuning path | +| Evaluation | Implemented, local-only | `training/evaluation.py` provides prompt cases, exact-match scoring, optional loss-based perplexity, qualitative table, base-vs-tuned comparison, and JSONL logging | +| Export tab | Partial | `ui/export_tab.py` builds GGUF download/conversion/quantization plans, lists exported files, and exposes existing exported files through a download output | +| Export planner | Implemented, non-executing | `training/export.py` detects llama.cpp tools, builds explicit commands, and does not run downloads/conversions | +| Reward evaluation | Implemented locally | `training/reward_eval.py` provides deterministic reward scoring, best-of-N selection, DPO pair generation, and LoRA-vs-base reward reports | +| Synthetic data generation | Implemented locally | `datasets/synthetic.py` provides deterministic generation, validation, quality filtering, augmentation, and JSONL export | +| Field notes | Partial | `ui/notes_tab.py` saves CSV, supports media paths/training flag, imports uncertain OCR predictions, emits field-note events, and exports JSONL/local HF Dataset files | +| Field note module | Implemented | CSV save, SQLite store, corrected/tag/training filters, JSONL export, local HF Dataset export via `datasets/field_notes.py` | +| OCR correction loop | Implemented locally | `datasets/ocr.py` loads local CSV/JSONL OCR predictions, filters by confidence threshold, imports uncertain rows to Field Notes, and exports corrected OCR JSONL | +| Tracking | Implemented with local fallback | `tracking/trackio_client.py` loads Trackio config, writes local JSONL traces, and calls Trackio when installed/enabled | +| Traces tab | Partial | `ui/traces_tab.py` previews app events, reads local trace rows, shows tracking status, and exports traces | +| Agent mode | Implemented locally, non-autonomous | `agent/runner.py` provides system prompt, deterministic research-plan-implement-verify trace, paper-to-code trace mode, safety gates, tool registry integration, JSONL trace save/export, and local HF Dataset-style export | +| Agent tab | Implemented locally | `ui/agent_tab.py` drafts task traces, paper-to-code traces, and exports trace files/datasets | +| Status tab | Implemented | `ui/status_tab.py` lists model config, backend status, local llama.cpp setup, LM Studio/OpenAI-compatible setup, SGLang setup, and Ollama list/pull planning | +| Tab-level error messages | Implemented | Chat, Vision, and Dataset tabs show status/error messages and emit `ui_error` trace events | +| Loading/progress states | Implemented | `ui/progress.py` applies full Gradio progress indicators to tab actions | +| Compact responsive layout | Implemented | `APP_CSS` constrains app width, keeps tabs scrollable, sizes touch targets, and adds mobile padding/type rules | +| Structure verification | Done | `scripts/verify_structure.ps1` passed | +| Unit tests | Passing | 187 unit/user-story tests pass | +| User-story tests | Passing | Included in the 187-test suite | +| Coverage | Passing | 68% line/branch coverage at current configured threshold | +| Performance tests | Passing | 2 lightweight performance tests pass | +| Playwright E2E | Passing with real response screenshots | Workbench Playwright captures a real local GGUF `llama-cpp-python` chat response; Plant Playwright with `RUN_REAL_MODEL_E2E=1` captures a real OpenBMB MiniCPM-V image result from `assets/plant_sample.jpg` | +| CI pipeline | Added, not run remotely | `.github/workflows/ci.yml` | +| Quality tooling | Passing | Tests, coverage, performance, ruff, mypy, pylint, bandit, and pip-audit pass through `scripts/run_quality.ps1` | +| Secrets and model-weight git policy | Implemented | `.gitignore` excludes env files, keys, caches, generated data/exports, and common model weight formats; policy has a unit test | +| Real model inference | Partial but materially verified | Verified paths: LM Studio/OpenAI-compatible text, llama.cpp CLI text, llama-cpp-python GGUF text, and OpenBMB MiniCPM-V Plant image inference. Remaining unverified paths: Ollama generation, SGLang server generation, llama.cpp MiniCPM-V mmproj vision, full Transformers text generation | +| Hugging Face Space deploy | Pushed, startup/build pending | Workbench pushed to `build-small-hackathon/workbench` at `6aafdc2083a9b82e9dca2cca5b87c3a1be05121b`; Plant pushed to `build-small-hackathon/plant_identification_tool` at `50897b3167a844b6a66ca0552b73a1791cdff926`; both include Python 3.10 compatibility fixes for HF Spaces | +| HF Space deployment helper | Implemented locally | `deployment/hf_space.py` and `scripts/plan_hf_space.py` validate required files, README Space metadata, Workbench/Plant remote status, and manual `hf` deployment commands | +| vLLM serving tab | Implemented locally, not locally verified | `models/vllm_runner.py` and `ui/vllm_tab.py` build explicit vLLM commands, check health, parse metrics, log benchmark metrics through local tracking, and use OpenAI-compatible chat when a server is running | +| Plant Discovery reference app | Implemented locally, no-model verified; Space wrapper added | `plant/` is a standalone template-built app with demo/no-model service for local tests, default OpenBMB MiniCPM-V mode, optional fine-tuned adapter mode, local species index, correction export, non-executing training plan, optional MCP tools, unit tests, HTTP smoke verification on port 7861, and `plant_space_app.py` for real-model Space launch | +| GitHub push | Done | GitHub remote `https://github.com/Ckal/codex.git`; commits pushed to `origin/main` | + +## Known Blockers + +- `python` is available on PATH as Python 3.13 in the current shell. The documented `.venv` was not + visible during the latest verification run, so tests ran against the global Python environment + after reinstalling/updating `requirements.txt`. +- App launch was verified by Playwright through `python app.py`; no long-running server is currently active. +- llama.cpp tools are installed at `C:\llama-b9587-bin-win-cuda-13.3-x64`; they are not on PATH, + so the app stores/uses the explicit `llama-server.exe` path. +- Export planning works, but actual conversion/quantization remains blocked until a specific export + run is requested and verified. +- A GGUF path is configured locally for `Llama-3.2-1B-Instruct-Q4_K_M.gguf`; direct + `llama-cli.exe` and `llama-cpp-python` generation both produced real text. +- Ollama is installed on PATH (`ollama version is 0.24.0`). `ollama pull openbmb/minicpm-v4.6` + succeeded and `ollama list` shows `openbmb/minicpm-v4.6:latest`; a tiny `ollama run` prompt + currently fails with a 500 model-load error, so real Ollama generation remains unverified. +- LM Studio/OpenAI-compatible setup is implemented and verified for `http://192.168.188.37:1234` + with served model override `llama-3.2-1b-instruct`. This local override is stored in ignored + `data/local_backends.yaml`. +- `transformers 5.10.2`, `torch`, and MiniCPM-V dependencies are installed. This conflicts with + the currently installed `sentence-transformers 3.4.1` requirement of `transformers<5.0.0`, so + future quality runs should watch for that dependency edge. +- MiniCPM-V Plant inference is verified with `assets/plant_sample.jpg`; the app also supports + bounded E2E inference through `PLANT_MAX_NEW_TOKENS=320` and `PLANT_AUTO_THINKING=0`. +- SGLang command planning, health, stop, and chat client code is implemented. SGLang remains an + optional local backend, not a root Space dependency, because the local Python 3.13 package index + cannot install the newer non-vulnerable SGLang server stack. +- vLLM command planning, health, metrics, benchmark logging, and chat client code is implemented, + but the `vllm` package/server is not installed or running in this workspace. +- LoRA training request planning is implemented, but real execution remains blocked until PEFT/TRL, + Transformers, Torch, and final hardware are approved and installed. +- Trackio is optional; local JSONL tracing works without the `trackio` package, but remote Trackio/HF + sync still needs package availability and credentials. +- LoRA dry-run planning works locally, but real training remains blocked until a final backend, + PEFT/TRL or SWIFT/LLaMA-Factory path, and hardware are chosen. +- Hugging Face dataset preview is optional and requires the external `datasets` package; the app + reports a clear status when it is not installed. +- Hugging Face Space deployment is pushed with Python 3.10 compatibility fixes for `StrEnum` and + UTC timestamps. Final run verification and smoke workflows remain open until rebuilt Spaces + report `RUNNING`. +- Node.js and npm are installed (`node v23.11.0`, `npm 10.9.2`). `npm install`, + `npm run e2e:install`, and `npm run e2e` pass. +- Plant Discovery OpenBMB MiniCPM-V inference is implemented as the default real mode, and + `plant_space_app.py` launches that mode for Space deployment. `RUN_REAL_MODEL_E2E=1` + Playwright passed and captured a real response screenshot. +- Plant Discovery fine-tuned adapter mode is implemented, but no trained plant adapter exists in + this workspace yet. +- Plant Discovery public Space mode still needs path/url hardening and screenshots. +- Full PRD implementation is not complete. There are still unchecked tasks in `docs/TASKS.md`. +- Current unchecked task count needs recounting after the latest Workbench/Space changes. Several PRD/ext PRD items still + need real local setup, credentials, hardware, product decisions, or hackathon submission artifacts. + +## Latest Local Verification + +- `.\scripts\run_tests.ps1` passed: 200 tests and 70% coverage. +- `.\scripts\run_quality.ps1` passed: tests, app smoke, 70% coverage, performance, Ruff, mypy, + Pylint, Bandit, and project-scoped pip-audit. +- Plant Discovery no-model HTTP smoke passed on `http://127.0.0.1:7861`; the process was stopped + after verification. +- `python scripts\plan_plant_training.py --corrected-examples 30` prints a non-executing SWIFT / + LLaMA-Factory adapter training plan; current environment is missing torch, transformers, PEFT, + TRL, and SWIFT. +- `pytest tests/unit/test_local_backend_config.py tests/unit/test_llama_cpp_service.py tests/unit/test_model_catalog.py tests/unit/test_service_factory.py -q` passed: 30 tests. +- `npm run e2e:workbench` passed and captured a visible local GGUF response through `llama-cpp-python`. +- `RUN_REAL_MODEL_E2E=1 PLANT_MAX_NEW_TOKENS=320 PLANT_AUTO_THINKING=0 npx playwright test tests/e2e/plant_real_model.spec.ts --config playwright.plant.config.ts --reporter=list` passed and captured a real MiniCPM-V response. +- `hf upload build-small-hackathon/workbench C:\tmp\workbench_space_payload . --repo-type space` pushed commit `6aafdc2083a9b82e9dca2cca5b87c3a1be05121b`. +- `hf upload build-small-hackathon/plant_identification_tool C:\tmp\plant_space_payload . --repo-type space` pushed commit `50897b3167a844b6a66ca0552b73a1791cdff926`. +- `hf spaces variables add` set `WORKBENCH_DEPLOYMENT=space` for Workbench and Plant; Plant also has `PLANT_MAX_NEW_TOKENS=320` and `PLANT_AUTO_THINKING=0`. +- `hf spaces info` showed Workbench in `APP_STARTING` and Plant in `BUILDING` on `zero-a10g`; final build/run verification remains pending. +- `pytest tests/unit tests/user_stories -q` passed earlier in this workstream: 197 tests. +- `node --version` returned `v23.11.0`; `npm --version` returned `10.9.2`. +- `ollama --version` returned `ollama version is 0.24.0`; `ollama list` returned `gemma4:latest`. +- `ollama pull openbmb/minicpm5-1b` failed because the registry manifest does not exist. +- `ollama pull openbmb/minicpm-v4.6` succeeded; `ollama run openbmb/minicpm-v4.6` failed with + a local 500 model-load error. +- `hf --version` returned `1.17.0`; `huggingface-cli --version` reports the legacy command is + deprecated and recommends `hf`. +- `hf auth whoami` failed with "Invalid user token"; Space push/build verification is blocked + until `hf auth login --force` is run with a fresh token. +- `llama-server --version` failed because `llama-server` is not on PATH. +- `git remote -v` shows `space-workbench` and `space-plant` remotes configured. +- Direct `ruff check .` passed; cache-write warnings were caused by OneDrive permissions. +- Direct `mypy . --no-incremental` passed when `MYPY_CACHE_DIR` was moved to `%TEMP%`. +- LM Studio `/v1/models` at `http://192.168.188.37:1234` returned + `text-embedding-nomic-embed-text-v1.5`, `qwen2.5-coder-3b-instruct`, and + `llama-3.2-1b-instruct`. +- LM Studio `/v1/chat/completions` returned a text response from `llama-3.2-1b-instruct`. +- Focused OCR callback and pipeline tests pass: 8 tests. +- Focused VINDEX/MCP tests pass: 13 tests. + +## Verification Commands + +Run these after installing Python: + +```powershell +.\scripts\verify_structure.ps1 +.venv\Scripts\python.exe --version +.venv\Scripts\Activate.ps1 +python -m pip install -r requirements.txt +python -m pip install -r requirements-dev.txt +.\scripts\run_tests.ps1 +.\scripts\run_quality.ps1 +python app.py +``` + +When the app starts, update this file and `docs/TASKS.md`. diff --git a/docs/PLANT_DISCOVERY_APP_PLAN.md b/docs/PLANT_DISCOVERY_APP_PLAN.md new file mode 100644 index 0000000000000000000000000000000000000000..d063b9c6ce1a6e24bc1e73b325e77a76abc6bf81 --- /dev/null +++ b/docs/PLANT_DISCOVERY_APP_PLAN.md @@ -0,0 +1,119 @@ +# Plant Discovery Reference App Plan + +This checklist turns the rough `plant/` sample into the first concrete app built around the +template. It is intentionally detailed so future apps can copy the pattern. + +## Product Story + +Plant Discovery helps a gardener, teacher, or field-note collector: + +1. Upload one or more plant images. +2. Get a structured plant identification. +3. Correct mistakes. +4. Browse a small field guide. +5. Export corrections as local training data. +6. Later fine-tune or evaluate a small vision model. + +This is stronger than a generic workbench for hackathon judging because it has a real user outcome. + +## Current Implementation Status + +- [x] Add `plant/` package. +- [x] Add standalone `plant/app.py`. +- [x] Add clean `plant/models.yaml`. +- [x] Add deterministic no-model demo service. +- [x] Add optional MiniCPM-V service adapter. +- [x] Add explicit `demo`, `openbmb`, and `finetuned` model modes. +- [x] Make OpenBMB MiniCPM-V the default real model mode. +- [x] Add fine-tuned adapter loading path through PEFT when configured. +- [x] Avoid hard `torch`/`transformers` imports at module import time. +- [x] Add plant schema/parser through `PlantID`. +- [x] Add local species index builder. +- [x] Add local folder loader. +- [x] Add field-note export to plant training JSONL. +- [x] Add focused Gradio UI with Identify, Field Guide, Corrections, and Stats. +- [x] Replace direct training execution with non-executing training plan. +- [x] Add optional pure tool functions and lazy MCP server builder. +- [x] Add non-executing plant training planner. +- [x] Add Plant model/training how-to. +- [x] Add plant unit tests. +- [x] Add no-model app build verification. + +## Remaining Work + +### P0 - Make The Demo Judgeable + +- [x] Run `python -m plant.app --no-model --port 7861`. +- [ ] Install Node.js/npm. +- [ ] Generate Playwright screenshots for the plant app. +- [ ] Add Plant Discovery screenshots to README. +- [ ] Add a 60-90 second demo script focused on plant correction. +- [x] Decide whether the hackathon Space should launch `app.py` or `plant/app.py`. +- [x] If using `plant/app.py` for Space, add a Space-specific entrypoint or README note. + +### P1 - Real Model Path + +- [ ] Install optional plant dependencies from `plant/requirements.txt`. +- [ ] Verify `PlantVisionService.dependency_report()` is fully available. +- [ ] Run one MiniCPM-V identification on a local public-safe plant image. +- [ ] Capture model ID, hardware, latency, and output JSON. +- [ ] Add a real-backend integration test profile that is skipped unless dependencies are present. +- [ ] Document that model weights are not downloaded on startup. + +### P2 - Data And Correction Loop + +- [ ] Add a tiny public-safe sample plant image folder under ignored local data. +- [ ] Add a sample `plant/data/plantnet_labels.json` cache or document how to create it. +- [ ] Add path allowlist before public Space deployment. +- [ ] Add JSONL schema documentation for exported corrections. +- [ ] Add validation for minimum correction count before recommending training. + +### P3 - Template Extraction + +- [ ] Identify reusable parts that should move from `plant/` to generic modules. +- [ ] Keep domain-specific prompt/schema/UI inside `plant/`. +- [ ] Consider `domains//` convention if more than one reference app is added. +- [ ] Add a cookiecutter-style checklist or scaffold script only after the second app proves the pattern. + +### P4 - Security And Public Mode + +- [ ] Add public/local mode config. +- [ ] In public mode, disable arbitrary local file paths. +- [ ] In public mode, disable arbitrary backend URLs. +- [ ] Add max upload size and allowed image extension checks. +- [ ] Add tests for path traversal and malformed image/data inputs. +- [ ] Add a plant-specific disclaimer: identification can be wrong; do not use for medical, edible, + or toxicity-critical decisions without expert verification. + +### P5 - Training And Evaluation + +- [ ] Collect at least 30 corrected examples before any LoRA experiment. +- [ ] Split exported corrections into train/eval. +- [ ] Add exact species-name evaluation. +- [ ] Add before/after model comparison. +- [ ] Run SWIFT or LLaMA-Factory locally only after dependencies and hardware are approved. +- [ ] Publish adapter only if it improves the evaluation set. +- [ ] Update `plant/models.yaml` with the real adapter repo. +- [ ] Verify `python -m plant.app --model-mode finetuned --port 7861`. + +## Recommended Next Command Sequence + +```powershell +.venv\Scripts\python.exe -m pytest tests/unit/test_plant_reference_app.py -q +.venv\Scripts\ruff.exe check plant tests/unit/test_plant_reference_app.py --no-cache +.venv\Scripts\python.exe -m mypy plant tests/unit/test_plant_reference_app.py --cache-dir "$env:TEMP\openbmb-workbench-mypy-cache" +.venv\Scripts\python.exe -m plant.app --no-model --port 7861 +.venv\Scripts\python.exe scripts\plan_plant_training.py --corrected-examples 30 +``` + +For the deployed Plant Identification Tool Space, use `plant_space_app.py`. It launches +OpenBMB MiniCPM-V mode and does not enable `--no-model`/demo mode. + +## Template Lessons From Plant Discovery + +- Domain app first screen should be the product, not the infrastructure. +- Demo/no-model mode is essential for tests and screenshots. +- Heavy model dependencies should be optional. +- Correction loops are more valuable than promising immediate fine-tuning. +- Tool/MCP functions should be pure and importable without a running server. +- Training should be a plan until data, hardware, and dependencies are verified. diff --git a/docs/PLANT_MODEL_AND_TRAINING_HOWTO.md b/docs/PLANT_MODEL_AND_TRAINING_HOWTO.md new file mode 100644 index 0000000000000000000000000000000000000000..c876b9dd43841a14c0ddb01f313abf740b7fc210 --- /dev/null +++ b/docs/PLANT_MODEL_AND_TRAINING_HOWTO.md @@ -0,0 +1,145 @@ +# Plant Model And Training How-To + +This answers three common questions: + +1. Does Plant Discovery use an LLM? +2. Does it use a specially trained plant model? +3. How do we train and then use one? + +## Current Model Modes + +Plant Discovery has three explicit modes. + +| Mode | Command | Uses an LLM/VLM? | Trained specifically for this app? | +| --- | --- | --- | --- | +| Demo | `python -m plant.app --model-mode demo --port 7861` | No | No | +| OpenBMB zero-shot | `python -m plant.app --model-mode openbmb --port 7861` | Yes, MiniCPM-V | No, uses base OpenBMB model | +| Fine-tuned adapter | `python -m plant.app --model-mode finetuned --port 7861` | Yes, MiniCPM-V plus adapter | Yes, after you train/publish an adapter | + +The app defaults to `openbmb` mode. The `--no-model` flag is an alias for demo mode and exists for +tests, screenshots, and machines without GPU/model dependencies. + +## What Runs Today + +- Demo mode is fully verified locally. +- OpenBMB mode is implemented but requires optional dependencies and model weights. +- Fine-tuned mode is implemented as an adapter-loading path, but there is no real adapter yet. + +This means the app can use a real OpenBMB VLM, but this workspace has not installed the heavy +`plant/requirements.txt` dependencies or downloaded model weights. + +## Step 1 - Use OpenBMB MiniCPM-V Zero-Shot + +Install optional plant dependencies: + +```powershell +.venv\Scripts\python.exe -m pip install -r plant\requirements.txt +``` + +Run the app with the real OpenBMB model path: + +```powershell +.venv\Scripts\python.exe -m plant.app --model-mode openbmb --port 7861 +``` + +Open `http://127.0.0.1:7861`, upload a public-safe plant image, and click `Identify plant`. + +Expected behavior: + +- The UI shows model status with `uses_llm: true`. +- The first identify action loads `openbmb/MiniCPM-V-4.6`. +- If confidence is low and auto-thinking is enabled, the app can try + `openbmb/MiniCPM-V-4.6-Thinking`. +- Output is parsed into the `PlantID` JSON schema. + +## Step 2 - Collect Corrections + +Use the app normally: + +1. Upload plant image. +2. Identify. +3. Add a human correction when the model is wrong. +4. Save correction. +5. Open `Corrections`. +6. Click `Export training JSONL`. + +The export path is: + +```text +data/plant_training.jsonl +``` + +Do not train on one or two examples. The current minimum recommendation is 30 corrected examples, +and more is better. + +## Step 3 - Plan Training + +Generate a non-executing training plan: + +```powershell +.venv\Scripts\python.exe scripts\plan_plant_training.py --corrected-examples 30 +``` + +This prints: + +- dependency availability, +- SWIFT command preview, +- LLaMA-Factory command preview, +- publish commands, +- command to run the trained adapter. + +The Gradio UI also exposes the same non-executing plan from the `Corrections` tab. + +## Step 4 - Train Locally + +Preferred path for MiniCPM-V vision LoRA is SWIFT or another tool that supports multimodal LoRA. +The generated SWIFT command looks like this: + +```powershell +swift sft --model openbmb/MiniCPM-V-4.6 --dataset data/plant_training.jsonl --lora_rank 16 --num_train_epochs 3 --per_device_train_batch_size 4 --gradient_accumulation_steps 4 --learning_rate 2.0e-4 --freeze_vit true --output_dir checkpoints/plant_lora +``` + +Run it only after: + +- optional dependencies are installed, +- GPU memory is sufficient, +- exported JSONL has been reviewed, +- private images/notes are removed, +- you accept that training may take time and disk space. + +## Step 5 - Publish Or Configure The Adapter + +After training, publish the adapter or keep it local. If publishing: + +```powershell +huggingface-cli repo create your-username/minicpm-v46-plant-lora --type model +huggingface-cli upload your-username/minicpm-v46-plant-lora checkpoints/plant_lora . +``` + +Then edit `plant/models.yaml`: + +```yaml +plant_vlm_finetuned: + base_model: openbmb/MiniCPM-V-4.6 + adapter_id: your-username/minicpm-v46-plant-lora +``` + +Run: + +```powershell +.venv\Scripts\python.exe -m plant.app --model-mode finetuned --port 7861 +``` + +## Important Honesty Rule + +Do not claim Plant Discovery uses a specially trained model until: + +- a real adapter exists, +- it is configured in `plant/models.yaml`, +- `--model-mode finetuned` has been run, +- evaluation shows it improves over the base OpenBMB model. + +Until then, the honest claim is: + +> Plant Discovery uses OpenBMB MiniCPM-V zero-shot, with a correction loop and documented path to +> fine-tune a plant adapter. diff --git a/docs/PRD_IMPLEMENTATION_MATRIX.md b/docs/PRD_IMPLEMENTATION_MATRIX.md new file mode 100644 index 0000000000000000000000000000000000000000..9f2493e83b092de9f44dcf2675b4c0917857cfc8 --- /dev/null +++ b/docs/PRD_IMPLEMENTATION_MATRIX.md @@ -0,0 +1,95 @@ +# PRD Implementation Matrix + +This file maps the main PRD and extension PRD to current implementation status. + +## Summary + +The full PRD and extension PRD are not fully implemented yet. + +Current state: + +- Foundation, docs, test policy, quality gates, CI, placeholder Gradio surfaces, and a Plant + Discovery reference app exist. +- Shared app state, service registries, local event logging, tab-level error events, and local + trace preview exist. +- Local llama.cpp settings, GGUF/mmproj pickers, and command generation exist without startup downloads. +- GGUF export planning exists with tool detection and explicit non-executing command plans. +- Local JSONL tracing and optional Trackio wrapper exist. +- Dataset statistics and local MCP tool functions exist. +- OCR correction loop exists locally for CSV/JSONL prediction imports into Field Notes. +- VINDEX integration boundary exists locally as non-executing MCP-style planning tools. +- Local non-autonomous agent mode exists with trace export. +- Real local model inference is partially implemented through llama.cpp, llama-cpp-python, Ollama, + OpenAI-compatible/LM Studio, SGLang, Transformers text, and MiniCPM vision services. Verified + local paths now include llama.cpp CLI text, llama-cpp-python GGUF text, LM Studio text, and + OpenBMB MiniCPM-V Plant image inference. The Status + tab includes llama.cpp setup, LM Studio/OpenAI-compatible setup, SGLang command/check/stop setup, + and Ollama local model listing plus explicit pull-command planning. LM Studio text generation is + live-verified with `llama-3.2-1b-instruct`; the other real backends still need local verification. +- `WORKBENCH_DEPLOYMENT=space` now hides placeholder backend choices and refuses placeholder/demo + service creation for deployed app paths. +- LoRA training execution, served MCP endpoint, deployment, and most extensions are not implemented. +- Placeholder services remain intentionally visible so the app never pretends to be real inference. + +## Main PRD + +| PRD Area | Status | Evidence / Next Step | +| --- | --- | --- | +| Purpose and design philosophy | Documented | `README.md`, `docs/ROADMAP.md` | +| Template architecture | Partial | Config-driven model catalog exists; `docs/TEMPLATE_HOWTO.md` and `plant/` show the first domain-app pattern | +| System architecture | Partial | `app.py`, `core/`, `models/`, `ui/`, `datasets/`, local app state/events | +| Model registry | Partial | `config/models.yaml`, `models/model_catalog.py`; includes GGUF and backend capability metadata | +| Five inference modes | Partial | llama.cpp, llama-cpp-python, OpenAI-compatible/LM Studio, and MiniCPM-V Plant image inference are locally verified; Ollama generation, SGLang server generation, vLLM server generation, and llama.cpp mmproj vision remain unverified | +| Trackio | Partial | Local traces, optional Trackio wrapper, and HF Space sync docs exist; credentials/package setup still missing | +| MCP layer | Partial | Local tool functions, Gradio-native MCP path metadata, `mcp_server=True` launch flag, and local invocation tests exist; full external client verification still missing | +| Training pipeline | Partial | `training/` package supports dry-run planning, non-executing LoRA request planning, export planning, exact-match/perplexity evaluation, and local logging; real PEFT/TRL execution missing | +| Export and quantization | Partial | `training/export.py` and Export tab plan downloads/conversion/quantization and expose existing exported files for download; execution still missing | +| Agent mode | Partial | Local deterministic task and paper-to-code trace loops exist with safety gates; autonomous execution and remote uploads missing | +| UI tabs | Partial | Tabs exist; Chat/Vision/Dataset/Field Notes/Status have behavior; Status includes SGLang setup; tab actions have Gradio progress indicators; Chat/Vision/Dataset have tab-level status/error messages; compact responsive CSS exists; several tabs are still placeholders | +| Field notes | Partial | CSV save, SQLite store, corrected/tag/training filters, media paths, OCR uncertain import, JSONL export, and local HF Dataset export exist; remote HF upload missing | +| Directory structure | Partial | Foundation exists; many PRD packages missing | +| Configuration schema | Partial | Model/training config plus ignored local backend config exists; validation is lightweight | +| Dependencies | Partial | Runtime/dev deps exist for scaffold; full model/training deps not added | +| Hackathon demo flow | Partial | `docs/HACKATHON_SUBMISSION.md` drafts story, user, demo flow, script, social post, and URLs; real backend and Space URL still missing | +| Corrections from PRD v1 | Documented in PRD | Not all implemented | +| Roadmap and extension points | Documented | `docs/ROADMAP.md`, `docs/TASKS.md` | + +## Extension PRD + +| Extension | Status | Evidence / Next Step | +| --- | --- | --- | +| vLLM serving tab | Implemented locally, not locally verified | `models/vllm_runner.py` and vLLM tab provide command planning, health checks, metrics parsing, benchmark logging, and OpenAI-compatible chat client; needs installed/running vLLM for real serving | +| Ollama quick-start | Partial | Service, UI backend selector, local model listing, explicit pull-command planning, and setup docs exist; local Ollama install/real model verification missing | +| Reward model eval | Implemented locally | `training/reward_eval.py` provides deterministic reward scoring, best-of-N, DPO pairs, and LoRA-vs-base reward reports | +| Synthetic data generation | Implemented locally | `datasets/synthetic.py` provides deterministic generation, validation, filtering, augmentation, and JSONL export | +| Paper-to-code agent | Implemented locally | Agent tab and `agent/runner.py` support paper input, research/plan/implementation/verify trace, and safety gates without autonomous execution | +| HF Spaces deploy | Partial | README metadata, deployment helper, command plan, required-file validation, Workbench/Plant target URLs, and remote/build status checks exist; HF auth/remote/push/build verification still missing | +| VINDEX integration | Implemented locally, execution disabled | `mcp_tools/vindex_tool.py` validates VINDEX methods, builds safe call plans, reports dependency/server status, and documents that actual edits require a verified local VINDEX install | +| OCR pipeline hook | Implemented locally | `datasets/ocr.py` and Field Notes tab support local OCR prediction loading, confidence thresholds, uncertain import, human correction, and corrected JSONL export | +| MiniCPM Desk-Pet | Not implemented | Needs persona schema/export | +| MiniCPM-o audio tab | Not implemented | Needs audio tab and omnimodal backend | +| Cross-extension wiring | Partial | OCR -> Field Notes -> Training, Synthetic Gen -> Reward Eval -> DPO, Agent -> Desk-Pet Persona, and HF Spaces -> Trackio are documented; remaining wiring depends on unimplemented runtime modules | + +## Quality Coverage + +Current verified gates: + +- Structure check passes. +- 187 unit/user-story tests pass. +- Coverage report passes at 68%, above the current 60% configured threshold. +- 2 lightweight performance tests pass. +- Ruff passes. +- Mypy passes. +- Pylint passes at 10/10. +- Bandit reports no issues. +- Pip-audit reports no known vulnerabilities in `.venv`. +- LM Studio `/v1/models` and `/v1/chat/completions` are verified locally for + `llama-3.2-1b-instruct`. +- Workbench and Plant Playwright screenshot flows pass through `npm run e2e`. +- CI workflow exists but has not run remotely. +- App launch has been verified locally through Playwright, but the server is not currently left running. + +## No Pretend-Done Rule + +Any row marked `Partial`, `Placeholder`, or `Not implemented` must not be described as complete. +When a row is implemented, update this file, `docs/TASKS.md`, and `docs/IMPLEMENTATION_STATUS.md`. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..869d5460dc467f5cbcdc73ab596fb564dc56a821 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,32 @@ +# Project Docs + +This folder is the project memory. Use it before asking Codex to make larger changes. + +## Start Here + +- [TASKS.md](TASKS.md) - checklist of hackathon tasks and subtasks. +- [IMPLEMENTATION_STATUS.md](IMPLEMENTATION_STATUS.md) - what is done, partial, or not started. +- [PRD_IMPLEMENTATION_MATRIX.md](PRD_IMPLEMENTATION_MATRIX.md) - exact PRD/ext implementation coverage. +- [ACCEPTANCE_CRITERIA.md](ACCEPTANCE_CRITERIA.md) - proof required before marking work done. +- [ROADMAP.md](ROADMAP.md) - hackathon path versus long-term PRD vision. +- [ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md](ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md) - hard + judge-oriented critique and second roadmap. +- [TEMPLATE_HOWTO.md](TEMPLATE_HOWTO.md) - how to build new domain apps from this template. +- [PLANT_DISCOVERY_APP_PLAN.md](PLANT_DISCOVERY_APP_PLAN.md) - detailed checklist for the first + reference app. +- [PLANT_MODEL_AND_TRAINING_HOWTO.md](PLANT_MODEL_AND_TRAINING_HOWTO.md) - model modes and + adapter training path for Plant Discovery. +- [PLANT_MODEL_AND_TRAINING_HOWTO.md](PLANT_MODEL_AND_TRAINING_HOWTO.md) - how Plant Discovery + uses OpenBMB MiniCPM-V and how to train an adapter. +- [USAGE.md](USAGE.md) - how to install, run, and use the app. +- [ARCHITECTURE.md](ARCHITECTURE.md) - how the code is organized and how pieces connect. +- [EXTENDING.md](EXTENDING.md) - how to add models, tabs, services, and features later. +- [HACKATHON_SUBMISSION.md](HACKATHON_SUBMISSION.md) - story, demo flow, video script, and submission checklist. + +## Source PRDs + +- [HF_PRD_v1.md](../HF_PRD_v1.md) - main technical PRD. +- [HF_PRD_ext.md](../HF_PRD_ext.md) - planned extensions. + +The PRDs describe the full vision. The task and status docs describe what is actually implemented +or planned for the hackathon MVP. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000000000000000000000000000000000000..a6293524677e9629a9a08d65c7f09249f76e2c07 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,49 @@ +# Roadmap + +The PRD is intentionally broad. This roadmap keeps the hackathon path separate from the larger +Workbench vision. + +## Must Ship For Hackathon + +1. Local app runs. +2. One real small model backend works. +3. User story is clear. +4. Field notes or demo evidence shows practical value. +5. Hugging Face Space is live. +6. README is clear enough for judges. +7. Demo video and social post are ready. + +## Should Ship If Time Allows + +1. Ollama quick-start. +2. llama.cpp GGUF mode. +3. Field notes JSONL export. +4. Trackio trace logging. +5. Better visual polish. +6. Architecture/report write-up. + +## Stretch After Hackathon + +1. LoRA training. +2. GGUF export and quantization. +3. MCP tools. +4. Agent mode. +5. Reward model evaluation. +6. Synthetic data generation. +7. OCR pipeline hook. +8. MiniCPM-o audio tab. +9. Desk-Pet persona export. + +## Current Recommended Path + +For a new Codex user, the best path is: + +```text +docs/TASKS.md + -> install Python + -> run placeholder app + -> add Ollama backend + -> push GitHub + -> push Hugging Face Space + -> polish submission +``` diff --git a/docs/ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md b/docs/ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md new file mode 100644 index 0000000000000000000000000000000000000000..cbacbf44be9b1b68637c35dfb71d32bbc85cad20 --- /dev/null +++ b/docs/ROADMAP_V2_CRITICAL_IMPROVEMENT_PLAN.md @@ -0,0 +1,361 @@ +# Roadmap V2 - Critical Hackathon Improvement Plan + +This document is intentionally hard on the project. It answers: "If a strong automated judge or +human judge reviews this repo, what will cost points, and what should be fixed first?" + +Sources checked on 2026-06-08: + +- Official Build Small Hackathon page: +- Current repo status: `docs/IMPLEMENTATION_STATUS.md` +- PRD coverage: `docs/PRD_IMPLEMENTATION_MATRIX.md` +- Task checklist: `docs/TASKS.md` + +## Executive Verdict + +This is not yet a SOTA hackathon entry. It is a serious engineering scaffold with good local +quality discipline, but it is not yet a sharp, shipped product. + +Current estimated rating: + +| Dimension | Score | Hard reason | +| --- | ---: | --- | +| Hackathon fit | 5/10 | It uses Gradio and small-model framing, but the user story is diluted by too many tabs. | +| Shipped product | 3/10 | No verified Hugging Face Space URL, no generated screenshots, no submitted demo media. | +| Real usefulness | 5/10 | Field Notes plus correction loop can be useful, but it needs one real user, one real workflow, and evidence. | +| Technical architecture | 6/10 | Modules are clean enough, but UI callbacks still orchestrate too much and global state is too central. | +| Security posture | 5/10 | Basic secret/model-weight policy exists, but URL/path inputs and future command execution need hardening. | +| Test discipline | 7/10 | Good Python quality gates and unit tests; missing executed browser E2E, visual checks, and real backend integration. | +| Design and UX | 4/10 | Functional Gradio layout, but not memorable, not guided, and not custom enough for an "Off-Brand" award. | +| SOTA small-model angle | 4/10 | Many backend planners exist, but there is no benchmarked small-model capability story yet. | + +Overall: **5/10 as a repo, 3.5/10 as a winning submission today.** + +The strongest path is to stop trying to show everything and make one narrow workflow excellent: + +> A local-first Field Notes Correction Workbench for collecting messy human observations, correcting +> small-model/OCR outputs, exporting training data, and showing how small models improve with human +> feedback. + +That can plausibly compete in Backyard AI, Field Notes, Sharing is Caring, Llama Champion, Tiny Titan, +OpenBMB, and Best Demo if executed cleanly. + +## What A GPT-Style Judge Will Likely Measure + +The official page does not publish a numeric rubric, but it does state the visible judging signals. +Assume an automated GPT judge will inspect the Space, README, demo, screenshots, repository, and +submission text for the following: + +1. **Track alignment** + - Backyard AI: specific real problem, real person used it, honest small-model fit, Gradio polish. + - Thousand Token Wood: delight, AI as the core experience, originality, Gradio polish. + +2. **Hard constraints** + - Gradio app. + - Hosted as a Hugging Face Space. + - Models at or below 32B parameters. + - Short demo video and social/social-style post. + +3. **Bonus quest evidence** + - Off the Grid: no cloud APIs. + - Well-Tuned: published fine-tuned model. + - Off-Brand: custom UI beyond default Gradio. + - Llama Champion: llama.cpp runtime. + - Sharing is Caring: public trace on the Hub. + - Field Notes: report or blog post about build and lessons. + +4. **Repo truthfulness** + - README claims match implementation. + - No hidden placeholders presented as working model features. + - Tests and status docs do not contradict the demo. + +5. **User outcome** + - A judge can answer: who is this for, what did they do before, what do they do now, and what got + measurably better? + +6. **Demo reliability** + - App loads quickly. + - One golden path works without local-only secrets. + - Screenshots and demo video show actual outputs, not empty controls. + +7. **Novelty and "small model" honesty** + - The app should show why a small model is enough or better. + - A tiny/small model should do something concrete, not merely be listed in config. + +## Current Critical Weaknesses + +### Product Weaknesses + +- The app reads like a toolbox, not a product. A judge will not spend long discovering value across + Chat, Vision, Dataset, Train, vLLM, Export, Field Notes, Traces, Agent, and Status. +- The project story is not yet visceral. "OpenBMB Local AI Workbench" sounds useful to builders, + but the hackathon asks for either a real person or a delightful experience. +- The strongest workflow is Field Notes/OCR/correction, but it is not yet the first screen or the + main README story. +- No demo screenshots have been generated yet because Node/npm is missing. +- No public Space URL means the current entry cannot satisfy the central shipping requirement. + +### Architecture Weaknesses + +- UI callbacks currently act as controllers. They validate inputs, call services, emit events, and + format output. This is acceptable for an MVP, but it will become brittle. +- `APP_STATE` is a global singleton. That is simple locally, but it is risky for multi-user Spaces, + test isolation, and future per-session traces. +- Service creation is centralized in `models/service_factory.py`, but there is no application-layer + facade for workflows such as "correct note -> export dataset -> evaluate -> log trace". +- There are many non-executing planners. That is honest, but it creates a product risk: the app can + look more complete than it is. +- There is no domain model for a complete demo run. The app records events, but not a first-class + `DemoSession`, `CorrectionSession`, or `EvaluationRun`. + +### Security Weaknesses + +- User-controlled URLs are accepted for local backends. This can become SSRF if deployed on a Space + and allowed to reach internal services. +- User-controlled local paths are accepted for datasets, GGUF files, OCR files, exports, and media. + This is useful locally, but dangerous on a public Space without path allowlists. +- Future command execution is planned. Command builders are currently non-executing, which is good, + but the moment execution is added, strict argument arrays, allowlists, and confirmation gates are + mandatory. +- MCP exposure is attractive for judges, but it can widen the attack surface. Tool inputs need + explicit schemas, rate limits, and audit logs before public use. +- Agent mode is non-autonomous today. If autonomy is added, shell/git/deploy/model-download actions + need hard policy gates. + +### Testing Weaknesses + +- Playwright E2E exists but is not run locally because Node/npm is missing. +- No browser screenshots have been generated yet for docs. +- No visual regression baseline. +- No real backend integration test for LM Studio, llama.cpp, Ollama, SGLang, vLLM, or MiniCPM Vision. +- Coverage is enough for the current threshold, but important UI code remains lightly covered. +- Security tests are thin: path allowlist, URL allowlist, SSRF prevention, file-size limits, and + malformed JSONL/CSV cases should be tested. + +### Design Weaknesses + +- The UI is still mostly default Gradio. That likely misses the Off-Brand award. +- The first screen does not guide a judge into a single demo path. +- There is no "Judge Mode" that preloads sample data, explains what to click, and proves the core + value in under two minutes. +- The README does not yet lead with screenshots, output examples, and the exact demo path. +- Status/setup surfaces are useful, but they should not dominate the judged experience. + +## Architecture Recommendation + +Do not add heavy enterprise architecture. Add a thin application layer. + +### Current Shape + +```text +ui tab callback -> model/dataset/training module -> event logging +``` + +This keeps files small, but the UI owns too much orchestration. + +### Recommended Shape + +```text +ui tab callback + -> controller + -> workflow facade + -> domain service / backend adapter + -> event/tracking/security policy +``` + +Add these modules: + +- `core/context.py` + - `AppContext` + - owns catalog, event bus, tracking client, local config, and security policy. + - replaces direct use of the global `APP_STATE` in new code. + +- `controllers/chat_controller.py` + - validates chat input. + - selects backend through an inference facade. + - emits trace events. + - returns UI-safe results. + +- `controllers/field_notes_controller.py` + - owns save/import/export correction workflows. + - produces documentation-ready demo summaries. + +- `controllers/dataset_controller.py` + - owns local/HF dataset preview with path policy checks. + +- `controllers/serving_controller.py` + - owns llama.cpp, SGLang, vLLM, Ollama command/status planning. + +- `workflows/correction_loop.py` + - `CorrectionLoopFacade` + - one golden path: import sample -> correct -> export JSONL -> evaluate -> trace. + +- `security/policies.py` + - URL allowlist for local backends. + - path allowlist for public Space mode. + - file size/type limits. + - execution policy for future command-running features. + +Keep the existing model services as adapters. Do not replace everything at once. + +### Singleton Decision + +The current `APP_STATE` singleton is tolerable for a local prototype. It is not ideal for a public +Space. Move toward: + +- `AppContext` created in `build_app()`. +- Per-session UI state through Gradio `gr.State` where user-specific data matters. +- Global event log only for public-safe aggregate events. +- Tests that create isolated contexts. + +## Roadmap V2 + +### P0 - Make It Shippable And Judgeable + +- [ ] Choose one final judged track: Backyard AI is recommended. +- [ ] Rename the public product around the user outcome, not the infrastructure. +- [ ] Make Field Notes Correction the first and primary demo path. +- [ ] Add a "Judge Mode" tab or first-screen panel with sample data and exact steps. +- [ ] Generate Playwright screenshots into `assets/e2e/`. +- [ ] Add screenshots to README and `docs/HACKATHON_SUBMISSION.md`. +- [ ] Deploy a working Hugging Face Space. +- [ ] Add the Space URL to README and submission docs. +- [ ] Record a 60-90 second demo video. +- [ ] Publish or draft the social post with Space, GitHub, and demo video links. + +Definition of done: a judge can open the Space and understand the product in under 30 seconds. + +### P1 - Prove One Real Small-Model Backend + +- [ ] Pick the final backend for the demo: LM Studio today, llama.cpp if badge is achievable. +- [ ] Run one verified local generation path end-to-end. +- [ ] Capture model ID, parameter count, runtime, hardware, and latency. +- [ ] Add a "Model Evidence" section to README. +- [ ] Add a tiny benchmark: response latency, tokens/sec if available, and sample output quality. +- [ ] If using OpenBMB, show exactly where the OpenBMB model is used. +- [ ] If claiming Llama Champion, install llama.cpp, select GGUF, and verify real text generation. + +Definition of done: the app has one undeniable real-model path, not just planners. + +### P2 - Make The Field Notes Workflow Real + +- [ ] Create a small public-safe example dataset. +- [ ] Add sample OCR predictions or sample model mistakes. +- [ ] Build "import -> correct -> export -> evaluate" as one guided workflow. +- [ ] Add before/after examples to README. +- [ ] Export corrected JSONL and trace data. +- [ ] Publish public-safe traces or dataset to Hugging Face. +- [ ] Write the Field Notes report/blog. + +Definition of done: the app demonstrates measurable improvement from human corrections. + +### P3 - Simplify The UI For Judges + +- [ ] Add a first-screen guided demo panel. +- [ ] Move advanced backend tabs behind an "Advanced setup" accordion or tab group. +- [ ] Add concise status badges: local-first, model size, backend, Space-ready, trace-ready. +- [ ] Add visual hierarchy and screenshots. +- [ ] Use a small custom CSS theme that fits the product story. +- [ ] Keep default Gradio controls where useful, but avoid an unstyled toolbox feel. + +Definition of done: the app feels intentional, not like a pile of tabs. + +### P4 - Add Controller And Facade Layer + +- [ ] Add `core/context.py` with injectable `AppContext`. +- [ ] Add `security/policies.py` with path and URL checks. +- [ ] Add `controllers/field_notes_controller.py`. +- [ ] Add `controllers/dataset_controller.py`. +- [ ] Add `controllers/chat_controller.py`. +- [ ] Add `controllers/serving_controller.py`. +- [ ] Add `workflows/correction_loop.py`. +- [ ] Refactor one tab at a time to call controllers instead of raw service modules. +- [ ] Keep old behavior covered by tests during each refactor. + +Definition of done: UI callbacks become thin and domain workflows are testable without Gradio. + +### P5 - Security Hardening + +- [ ] Add public/local mode setting. +- [ ] In public Space mode, restrict backend URLs to an allowlist or disable arbitrary URL checks. +- [ ] Restrict file paths to repo-local safe directories. +- [ ] Add max file size and supported extension checks. +- [ ] Add tests for path traversal attempts. +- [ ] Add tests for SSRF-style URLs. +- [ ] Add tests for malformed CSV/JSONL/OCR inputs. +- [ ] Add clear privacy labels for traces and field notes. +- [ ] Ensure demo data contains no secrets, private prompts, or hidden reasoning. + +Definition of done: the public Space cannot be used to probe internal URLs or arbitrary files. + +### P6 - Real Integration And E2E Tests + +- [ ] Install Node.js/npm. +- [ ] Run `npm install`, `npm run e2e:install`, and `npm run e2e`. +- [ ] Commit generated screenshot assets that are safe for docs. +- [ ] Add Playwright to CI if Node is available. +- [ ] Add one live-backend integration profile for LM Studio/OpenAI-compatible. +- [ ] Add optional integration profiles for llama.cpp, Ollama, SGLang, and vLLM. +- [ ] Add visual smoke checks for mobile and desktop. + +Definition of done: the demo path is proven in a browser and documented with images. + +### P7 - SOTA Stretch + +- [ ] Fine-tune or publish a tiny adapter if time allows. +- [ ] Publish a public-safe correction dataset or trace dataset. +- [ ] Add a simple reward/evaluation leaderboard for base vs corrected/fine-tuned outputs. +- [ ] Add a model card or dataset card with limitations. +- [ ] Show why the small model is good enough: privacy, latency, cost, offline use, or local control. +- [ ] If using llama.cpp, publish exact GGUF/runtime instructions. + +Definition of done: the project has a defensible small-model contribution, not just a UI. + +## What To Cut + +Cut or hide these from the judged first impression unless they support the demo: + +- vLLM tab. +- SGLang controls. +- Training execution promises. +- VINDEX boundary. +- Desk-Pet backlog. +- MiniCPM-o audio backlog. +- Generic agent mode, unless it directly produces the public trace badge. + +Keep them in docs as extension points, but do not make the judge click through them first. + +## Better Tool Vision + +A better tool is not "all PRD features implemented." A better tool is: + +1. **Specific** + - Helps one person correct and turn messy field observations into useful training/eval data. + +2. **Local-first** + - Runs with a small local model and explains why that matters. + +3. **Evidence-driven** + - Shows before/after outputs, corrected examples, traces, and screenshots. + +4. **Safe** + - Does not leak private notes, probe arbitrary URLs, or execute commands unexpectedly. + +5. **Small but complete** + - One polished path beats ten partial tabs. + +## Brutal Final Advice + +If the deadline is close, do not chase full PRD completion. The PRD is too broad for a winning +hackathon submission. Win by making one real, visual, end-to-end story undeniable. + +Priority order: + +1. Space URL. +2. One real model path. +3. Field Notes golden workflow. +4. Screenshots and demo video. +5. README story with evidence. +6. Security hardening for public mode. +7. Controller/facade cleanup. + +Everything else is secondary. diff --git a/docs/TASKS.md b/docs/TASKS.md new file mode 100644 index 0000000000000000000000000000000000000000..da8cd8f7348203fadd9919d64a47f876c1a6ef69 --- /dev/null +++ b/docs/TASKS.md @@ -0,0 +1,530 @@ +# Full Task Checklist + +This is the shared task list for you and Codex. It covers the hackathon MVP, the main PRD, and +the extension PRD. A task is complete only when the matching acceptance criteria are met and +`docs/IMPLEMENTATION_STATUS.md` is updated. + +## Legend + +- `[x]` done and documented +- `[~]` partially implemented or placeholder exists +- `[ ]` not started +- `[blocked]` blocked by missing local setup, credentials, hardware, or external decision + +## Phase 0 - Project Memory And Setup + +- [x] Add root `README.md`. +- [x] Add root `AGENTS.md`. +- [x] Add `.gitignore`. +- [x] Add `requirements.txt`. +- [x] Add `docs/` folder. +- [x] Add docs index. +- [x] Add full task checklist. +- [x] Add implementation status doc. +- [x] Add usage guide. +- [x] Add architecture guide. +- [x] Add extension guide. +- [x] Add acceptance criteria. +- [x] Add roadmap. +- [x] Add critical judge-oriented improvement roadmap. +- [x] Add template how-to for building new domain apps. +- [x] Add Plant Discovery reference app checklist. +- [x] Add Plant Discovery model and training how-to. +- [x] Add PRD implementation matrix. +- [x] Add test folder. +- [x] Add user-story test folder. +- [x] Add dev requirements. +- [x] Add Python quality config. +- [x] Add test runner script. +- [x] Add quality runner script. +- [x] Add CI workflow. +- [x] Add coverage gate. +- [x] Add performance test script. +- [x] Install Python 3.11+. +- [x] Verify `python --version`. +- [x] Create `.venv`. +- [x] Install dependencies. +- [x] Run `python app.py`. +- [x] Capture screenshot or note local URL. + +## Phase 1 - Hackathon Definition + +- [x] Choose track: Backyard AI or Thousand Token Wood. +- [x] Write one-sentence project story. +- [x] Define target user. +- [x] Define measurable user benefit. +- [x] Decide final model family and model IDs. +- [x] Confirm every model is <= 32B parameters. +- [x] Decide local-first badge target. +- [x] Decide llama.cpp badge target. +- [x] Decide open trace badge target. +- [x] Decide field notes/report badge target. +- [x] Write final demo flow. +- [x] Write demo video script. +- [x] Write social post draft. +- [ ] Add final submission checklist with exact URLs. + +## Phase 2 - MVP Gradio App + +- [x] Add `app.py`. +- [x] Add Gradio `Blocks` shell. +- [x] Add model config loader. +- [x] Add model metadata display. +- [x] Add Chat tab. +- [x] Add Vision tab. +- [x] Add Dataset tab placeholder. +- [x] Add Train tab placeholder. +- [x] Add Export tab placeholder. +- [x] Add Field Notes tab. +- [x] Add placeholder text service. +- [x] Add placeholder vision service. +- [x] Add Traces tab placeholder. +- [x] Add Agent tab placeholder. +- [x] Add Status tab placeholder. +- [x] Add PowerShell structure verification script. +- [x] Run structure verification script. +- [x] Run app locally. +- [x] Fix local launch errors found so far. +- [x] Add screenshot capture path to docs or README. +- [x] Add first demo GIF/video plan. + +## Phase 3 - Config-Driven Model Registry + +- [x] Add `config/models.yaml`. +- [x] Add text model entry for MiniCPM5-1B. +- [x] Add vision model entry for MiniCPM-V-4.6. +- [x] Add omnimodal model entry for MiniCPM-o-4.5. +- [x] Add typed `ModelInfo`. +- [x] Add `load_model_catalog()`. +- [x] Add `model_choices()`. +- [x] Add `model_summary()`. +- [x] Add MiniCPM5-1B-Thinking config. +- [x] Add MiniCPM4.1-8B config. +- [x] Add MiniCPM-V-4.6-Thinking config. +- [x] Add GGUF metadata in config. +- [x] Add backend capability metadata. +- [x] Add lightweight catalog validation helper. +- [x] Show warnings for models over 32B parameters. + +## Phase 4 - Core Architecture + +- [x] Add `core/events.py`. +- [x] Add `EventType`. +- [x] Add `Event`. +- [x] Add `EventBus`. +- [x] Add `core/registry.py`. +- [x] Add generic `Registry`. +- [x] Add global app state. +- [x] Register model services in a service registry. +- [x] Emit inference events from UI. +- [x] Emit field note events. +- [x] Add lightweight logging. +- [x] Add unit tests for config and registry. + +## Phase 5 - Testing And Quality + +- [x] Add `tests/unit/`. +- [x] Add `tests/user_stories/`. +- [x] Add model catalog unit tests. +- [x] Add field notes unit tests. +- [x] Add new-user user-story test. +- [x] Add `requirements-dev.txt`. +- [x] Add `pyproject.toml`. +- [x] Add `scripts/run_tests.ps1`. +- [x] Add `scripts/run_quality.ps1`. +- [x] Run unit and user-story tests. +- [x] Install dev quality tools. +- [x] Run `ruff`. +- [x] Run `mypy`. +- [x] Run `pylint`. +- [x] Run `bandit`. +- [x] Run `pip-audit`. +- [x] Add rule: failing bug/check requires a new or updated test. +- [x] Add coverage report. +- [x] Add lightweight performance tests. +- [x] Add CI pipeline. +- [x] Add Playwright or equivalent browser e2e test after Gradio runs. +- [ ] Add tests for each real backend as it is implemented. +- [x] Add tests for backend service selection. +- [x] Add tests for Ollama unavailable path. +- [x] Add tests for llama.cpp unavailable path and command building. +- [x] Add tests for llama-cpp-python unavailable path. +- [x] Add tests for OpenAI-compatible/LM Studio unavailable and request paths. + +## Phase 6 - Local Inference Backends + +- [x] Choose first real backend. +- [x] Add backend selector in UI. +- [x] Add model status panel. +- [x] Add explicit model load button. +- [x] Ensure no model weights download on startup. + +### Ollama Backend + +- [x] Confirm Ollama is installed. +- [x] Add `models/ollama_service.py`. +- [x] Add local model list. +- [x] Add pull model command with explicit user action. +- [x] Add text chat through Ollama. +- [x] Add vision chat through Ollama when supported. +- [x] Document Ollama setup. + +### llama.cpp Backend + +- [x] Confirm llama.cpp tools are installed. +- [x] Add `models/llama_cpp_service.py`. +- [x] Add `models/llama_cpp_python_service.py`. +- [x] Add GGUF file picker. +- [x] Add `llama-server` launch command builder. +- [x] Add health check. +- [x] Add text generation through server. +- [x] Add vision `mmproj` support metadata. +- [x] Document llama.cpp setup. + +### llama-cpp-python Backend + +- [x] Add optional Python binding service. +- [x] Add backend selector support. +- [x] Install `llama-cpp-python` locally. +- [x] Configure local GGUF path. +- [x] Verify real text generation through Python binding. +- [x] Decide whether to keep Python binding as fallback or primary local path. + +### Transformers Backend + +- [x] Add `models/transformers_text.py`. +- [x] Add `AutoModelForCausalLM` loading for text models. +- [x] Add tokenizer loading. +- [x] Add explicit trust-remote-code handling. +- [x] Add device/dtype settings. +- [x] Add streaming generation. +- [x] Document hardware expectations. + +### OpenAI-Compatible / LM Studio Backend + +- [x] Add `models/openai_compatible_service.py`. +- [x] Add backend selector support. +- [x] Add local base URL and served-model-name config. +- [x] Add Status tab setup and reachability check. +- [x] Add text chat through OpenAI-compatible `/v1/chat/completions`. +- [x] Document LM Studio setup. +- [x] Verify real text generation through LM Studio. + +### MiniCPM Vision Backend + +- [x] Add `models/minicpm_vision.py`. +- [x] Use `AutoModelForImageTextToText`. +- [x] Use `AutoProcessor`. +- [x] Add image prompt formatting. +- [x] Add thinking-mode toggle mapping. +- [x] Add video support plan. + +### SGLang Backend + +- [x] Add `models/sglang_runner.py`. +- [x] Add server start/stop. +- [x] Add MiniCPM5 tool parser config. +- [x] Add health check. +- [x] Add chat endpoint client. +- [x] Install `sglang` locally. + +## Phase 7 - UI Tabs From Main PRD + +- [x] Chat tab placeholder. +- [x] Vision tab placeholder. +- [x] Dataset tab placeholder. +- [x] Train tab placeholder. +- [x] Export tab placeholder. +- [x] Field Notes tab minimal save. +- [x] Add Traces tab with local event preview. +- [x] Add Agent tab placeholder. +- [x] Add model/backend status tab or panel. +- [x] Add settings panel. +- [x] Add tab-level error messages. +- [x] Add loading/progress states. +- [x] Add compact responsive layout review. + +## Phase 8 - Dataset Layer + +- [x] Add `datasets/` package. +- [x] Add local CSV loader. +- [x] Add local JSONL loader. +- [x] Add Hugging Face dataset loader. +- [x] Add dataset schema preview. +- [x] Add split selector. +- [x] Add row count and sample preview. +- [x] Add dataset statistics tool. +- [x] Emit `DATASET_LOADED` event. +- [x] Document dataset formats. + +## Phase 9 - Field Notes And Correction Loop + +- [x] Save field notes to CSV. +- [x] Move field note logic out of UI into `datasets/field_notes.py`. +- [x] Add `FieldNote` dataclass. +- [x] Add SQLite-backed store. +- [x] Add JSONL export. +- [x] Add local HF Dataset export. +- [x] Add corrected-only filter. +- [x] Add tags filter. +- [x] Add image path support. +- [x] Add video path support. +- [x] Add use-for-training flag. +- [x] Add docs for correction loop. + +## Phase 10 - Training Pipeline + +- [x] Add training config placeholder. +- [x] Add training UI placeholder. +- [x] Add `training/` package. +- [x] Add LoRA text trainer. +- [x] Add LoRA config parser. +- [ ] Add PEFT/TRL dependencies when ready. +- [x] Add training dry-run validation. +- [x] Add local checkpoint output. +- [x] Add Trackio integration. +- [x] Add evaluation after training. +- [x] Add LoRA vs base comparison. +- [x] Add vision fine-tuning plan using SWIFT or LLaMA-Factory. +- [x] Document training hardware requirements. + +## Phase 11 - Evaluation + +- [x] Add `training/evaluation.py`. +- [x] Add simple prompt test set. +- [x] Add exact-match metric. +- [x] Add qualitative eval table. +- [x] Add perplexity metric where appropriate. +- [x] Add base vs tuned comparison. +- [x] Log eval results. +- [x] Document evaluation method. + +## Phase 12 - Export And Quantization + +- [x] Add export UI placeholder. +- [x] Add `training/export.py`. +- [x] Add official GGUF download path. +- [x] Add local HF-to-GGUF conversion path. +- [x] Add quantization selector. +- [x] Add llama.cpp tool detection. +- [x] Add exported file listing. +- [x] Add download link in UI. +- [x] Document GGUF export. + +## Phase 13 - Trackio Tracing + +- [x] Add `tracking/` package. +- [x] Add Trackio config. +- [x] Add `trackio.init()`. +- [x] Add `trackio.log()`. +- [x] Add `trackio.finish()`. +- [x] Log inference events locally. +- [x] Log dataset events locally. +- [x] Log training metrics. +- [x] Add Traces tab. +- [x] Add HF Space sync docs. + +## Phase 13 - MCP Layer + +- [x] Decide MCP path: Gradio native, `gradio.Server` +- [x] Add MCP tools module. +- [x] Add dataset stats tool. +- [x] Add HF search tool. +- [x] Add safe calculator tool. +- [x] Add model inference tool. +- [x] Expose tools through selected MCP path. +- [x] Document MCP endpoint. +- [x] Verify endpoint locally. + +## Phase 14 - Agent Mode + +- [x] Add `agent/` package. +- [x] Add agent system prompt. +- [x] Add research-plan-implement loop placeholder. +- [x] Add tool registry integration. +- [x] Add session trace logging. +- [x] Add Agent tab. +- [x] Add trace export to JSONL. +- [x] Add local HF Dataset export for traces. +- [x] Document limitations. + +## Phase 15 - Hugging Face Space Deployment + +- [x] Install/verify `huggingface_hub`. +- [x] Login with `hf auth login`. +- [ ] Create Space. +- [x] Add Space README metadata if needed. +- [x] Add Space remote. +- [x] Push to Space. +- [ ] Verify Space builds. +- [ ] Add Space URL to README. +- [x] Document hardware choice. +- [x] Document model download behavior. + +## Phase 16 - GitHub + +- [x] Create GitHub repo. +- [x] Add GitHub remote. +- [x] Commit initial project. +- [x] Push to GitHub. +- [x] Add GitHub URL to README. +- [ ] Add issue checklist or project board if desired. + +## Phase 17 - Hackathon Submission Package + +- [x] Finalize app name. +- [x] Finalize track. +- [x] Verify Gradio app polish. +- [x] Verify model-size compliance. +- [ ] Verify Space URLs. +- [x] Verify GitHub URL. +- [ ] Record demo video. +- [ ] Publish social post. +- [ ] Add field notes/report link. +- [ ] Submit before June 15, 2026. + +## Extension PRD Backlog + +### vLLM Serving Tab + +- [x] Add vLLM runner. +- [x] Add vLLM start/stop UI. +- [x] Add OpenAI-compatible client. +- [x] Add metrics parsing. +- [x] Add Trackio benchmark logging. + +### Ollama Quick-Start + +- [x] Add Ollama pull/list UI. +- [x] Add Ollama chat service. +- [x] Add Ollama vision service. +- [x] Add setup docs. + +### Llama.cpp Champion Path + +- [x] Add llama.cpp backend selection. +- [x] Add llama.cpp service. +- [x] Add llama-cpp-python service. +- [x] Add llama.cpp status check. +- [x] Install llama.cpp locally. +- [x] Download/pick GGUF model. +- [x] Verify real text generation. +- [ ] Verify MiniCPM-V mmproj flow. + +### Reward Model Eval + +- [x] Add reward evaluator. +- [x] Add best-of-N generation. +- [x] Add DPO pair generation. +- [x] Add LoRA vs base reward report. + +### Synthetic Data Generation + +- [x] Add synthetic generator. +- [x] Add JSON validation. +- [x] Add quality filters. +- [x] Add augmentation flow. +- [x] Add dataset save/export. + +### Paper-To-Code Agent + +- [x] Add paper input UI. +- [x] Add research phase. +- [x] Add plan phase. +- [x] Add implementation trace. +- [x] Add safety gates. + +### HF Spaces Deploy Tool + +- [x] Add deployment helper script. +- [x] Add Space creation docs. +- [x] Add remote validation. +- [x] Add build status checks. + +### VINDEX Integration + +- [x] Define integration boundary. +- [x] Add tool stub. +- [x] Add verification report. +- [x] Document dependency. + +### OCR Pipeline Hook + +- [x] Add OCR loader. +- [x] Add confidence threshold. +- [x] Add uncertain prediction import. +- [x] Add correction UI. +- [x] Add corrected export. + +### MiniCPM Desk-Pet + +- [ ] Add persona data schema. +- [ ] Add persona training plan. +- [ ] Add Desk-Pet export plan. +- [ ] Add docs. + +### MiniCPM-o Audio Tab + +- [ ] Add audio tab. +- [ ] Add microphone input. +- [ ] Add omnimodal service. +- [ ] Add TTS plan. +- [ ] Add streaming plan. + +### Cross-Extension Wiring + +- [x] Document OCR -> Field Notes -> Training. +- [x] Document Synthetic Gen -> Reward Eval -> DPO. +- [x] Document Agent -> Desk-Pet Persona. +- [x] Document HF Spaces -> Trackio. + +## Phase 18 - Template And Reference Apps + +### Template How-To + +- [x] Document branch strategy for new domain apps. +- [x] Document required domain app file contract. +- [x] Document schema, service, loader, UI, tools, tests, and docs pattern. +- [x] Document no-model/demo-mode requirement. +- [x] Document correction-loop-first workflow. +- [x] Document optional training and real-model verification steps. +- [x] Document security requirements for public Space mode. + +### Plant Discovery Reference App + +- [x] Add `plant/` package. +- [x] Add standalone Plant Discovery Gradio entrypoint. +- [x] Add clean plant model/domain config. +- [x] Add deterministic no-model plant service. +- [x] Add optional MiniCPM-V plant service adapter. +- [x] Make OpenBMB MiniCPM-V the default real model mode. +- [x] Add explicit demo/openbmb/finetuned runtime modes. +- [x] Add optional fine-tuned adapter loading path. +- [x] Keep optional model dependencies lazy. +- [x] Add plant structured result schema and parser. +- [x] Add species index builder. +- [x] Add local image-folder loader. +- [x] Add field-note correction export to plant training JSONL. +- [x] Add focused Identify, Field Guide, Corrections, and Stats UI. +- [x] Replace direct training execution with non-executing training plan. +- [x] Add optional plant tool functions with lazy MCP server construction. +- [x] Add non-executing plant training planner. +- [x] Add `scripts/plan_plant_training.py`. +- [x] Add Plant Discovery unit tests. +- [x] Verify no-model app shell builds. +- [x] Run Plant Discovery as a long-running local app. +- [x] Generate Plant Discovery screenshots. +- [x] Add Plant Discovery screenshots to README/docs. +- [x] Decide whether hackathon Space launches root workbench or Plant Discovery app. +- [x] Verify real MiniCPM-V plant identification with optional dependencies. +- [ ] Train or configure a real Plant Discovery adapter. +- [ ] Verify `--model-mode finetuned` with the real adapter. +- [ ] Add public-mode file/path/url hardening before Space deployment. + +## Ongoing Maintenance + +- [x] Update docs after every implemented feature. +- [x] Keep `IMPLEMENTATION_STATUS.md` current. +- [x] Keep unchecked tasks visible. +- [x] Keep secrets and model weights out of git. +- [x] Re-run local app after code changes. diff --git a/docs/TEMPLATE_HOWTO.md b/docs/TEMPLATE_HOWTO.md new file mode 100644 index 0000000000000000000000000000000000000000..ab6807e3ac6b8bf76915c383585e87041e65f875 --- /dev/null +++ b/docs/TEMPLATE_HOWTO.md @@ -0,0 +1,207 @@ +# Template How-To: Build A New Domain App + +This repository is a local-first Gradio AI app template. The base workbench provides shared +patterns for model configuration, field notes, tracking, export planning, tests, docs, and +deployment. A domain app is a focused product built around those patterns. + +Use `plant/` as the first reference domain app. + +## Core Principle + +Do not start by training a model. Start by shipping a useful zero-shot or demo-mode workflow: + +```text +domain idea + -> user story + -> schema + -> model choice + -> focused UI + -> correction loop + -> export data + -> optional fine-tune + -> deploy and document +``` + +Training is a later optimization after you have corrected examples and a reason to tune. + +## Recommended Branch Flow + +1. Keep `main` as the reusable template. +2. Create a branch for each app: + + ```powershell + git checkout -b plant-discovery-app + ``` + +3. Build the app under a domain folder such as `plant/`, `invoice/`, `recipe/`, or `field_notes/`. +4. Keep domain-specific heavy requirements in `/requirements.txt`. +5. Merge reusable improvements back into `main` only after they are generic. + +## Domain App File Contract + +Each generated app should have these files: + +```text +/ + __init__.py + app.py # standalone Gradio entrypoint + models.yaml # domain config, model IDs, data sources, training defaults + _service.py # optional real model adapter plus demo/no-model fallback + _loader.py # data loading, schema normalization, export rows + _tab.py # focused Gradio UI + _tools.py # optional MCP/local tools with no hard optional imports + requirements.txt # optional heavy dependencies for this app only +``` + +Add tests under: + +```text +tests/unit/test__reference_app.py +``` + +Add docs under: + +```text +docs/_APP_PLAN.md +``` + +## Step-By-Step Build Process + +### 1. Define The Product + +- [ ] Pick one user. +- [ ] Pick one job they need done. +- [ ] Write one sentence: "This app helps X do Y without Z." +- [ ] Choose one golden path that works in under two minutes. +- [ ] Decide whether the app is a standalone product or a tab inside the workbench. +- [ ] Decide whether it must run on a public Hugging Face Space. + +Example: + +> Plant Discovery helps gardeners identify a plant from a photo, correct mistakes, and export +> local training examples without sending private field notes to a cloud API. + +### 2. Define The Domain Schema + +- [ ] Create a dataclass for the structured output. +- [ ] Include confidence and model metadata. +- [ ] Include a `to_dict()` method for Gradio JSON. +- [ ] Add a robust parser for model responses. +- [ ] Add tests for valid JSON, fenced JSON, trailing commas, and unparseable text. + +Plant example: `PlantID` in `plant/plant_service.py`. + +### 3. Pick The Model + +- [ ] Pick a small model at or below 32B parameters. +- [ ] Document the exact model ID. +- [ ] Add model metadata to `/models.yaml`. +- [ ] Avoid loading weights on startup. +- [ ] Add a deterministic demo/no-model service for screenshots and tests. +- [ ] Add an unavailable-path response when optional packages are missing. +- [ ] Add explicit runtime modes such as `demo`, `base-model`, and `finetuned`. +- [ ] Do not claim a fine-tuned model until a real adapter/checkpoint is configured and verified. + +For vision apps, start with a VLM such as MiniCPM-V. For text apps, start with a small instruct +model through LM Studio, Ollama, llama.cpp, or Transformers. + +### 4. Build The Focused UI + +- [ ] Make the first screen the golden path, not a generic dashboard. +- [ ] Add only the controls needed for the user story. +- [ ] Keep advanced setup behind a secondary tab or accordion. +- [ ] Add visible status messages. +- [ ] Add structured JSON output for debugging and reproducibility. +- [ ] Add correction capture if model output can be wrong. +- [ ] Add screenshots through Playwright after the UI is stable. + +### 5. Add The Correction Loop + +- [ ] Save user corrections locally. +- [ ] Reuse `datasets.field_notes.FieldNoteStore` where possible. +- [ ] Mark training-ready rows explicitly. +- [ ] Export JSONL without starting training. +- [ ] Add tests for save, filter, and export. + +### 6. Add Data Loaders + +- [ ] Support a small local demo dataset. +- [ ] Support domain data from local folders or CSV/JSONL. +- [ ] Keep Hugging Face dataset loading optional and explicit. +- [ ] Do not download large datasets on startup. +- [ ] Normalize every source into one training row schema. +- [ ] Add loader tests with temporary local files. + +### 7. Add Optional Tools + +- [ ] Keep MCP/tool imports optional. +- [ ] Tool functions should work locally without starting a server. +- [ ] Add `build_mcp_server()` only if `mcp` is installed. +- [ ] Avoid direct shell execution from tools. +- [ ] Return command plans rather than running commands. +- [ ] Add tests for pure tool functions. + +### 8. Add Training Plans + +- [ ] Start with a non-executing training plan. +- [ ] Include required dependencies, hardware notes, and command preview. +- [ ] Require enough corrected examples before recommending training. +- [ ] Keep real training as a separate local command or approved action. +- [ ] Add evaluation before/after tuning. +- [ ] Add a small script that prints the training plan as JSON. + +### 9. Add Security Guardrails + +- [ ] Escape model text rendered as HTML. +- [ ] Restrict file paths in public Space mode. +- [ ] Disable arbitrary backend URL checks in public Space mode. +- [ ] Do not execute subprocesses from Gradio callbacks. +- [ ] Keep tokens, private data, model weights, and exports out of git. +- [ ] Add tests for path traversal and malformed inputs when public deployment is planned. + +### 10. Verify The App + +Minimum local verification: + +```powershell +.venv\Scripts\python.exe -m pytest tests/unit/test__reference_app.py -q +.venv\Scripts\ruff.exe check tests/unit/test__reference_app.py --no-cache +.venv\Scripts\python.exe -m mypy tests/unit/test__reference_app.py --cache-dir "$env:TEMP\openbmb-workbench-mypy-cache" +.venv\Scripts\python.exe -c "from .app import build_app; app=build_app(no_model=True); print(type(app).__name__)" +``` + +Before claiming it works: + +- [ ] Run the standalone app. +- [ ] Generate screenshots. +- [ ] Add screenshot links to README/docs. +- [ ] Run full quality checks. +- [ ] Commit and push. + +## When To Integrate Into The Main Workbench + +Keep the domain app standalone if: + +- it has its own brand/story, +- it needs a focused judging experience, +- it has domain-specific dependencies, +- it should become a Hugging Face Space. + +Add it to the main workbench only if: + +- it is a generic reusable tab, +- it does not add heavy dependencies, +- it strengthens the template for all future apps. + +For the hackathon, standalone `plant/` is the better route because judges need one clear product. + +## What "Done" Means For A Domain App + +- [ ] Standalone no-model app builds. +- [ ] Optional real model adapter is documented and lazy-loaded. +- [ ] Golden path has tests. +- [ ] Corrections export to training data. +- [ ] Training is planned, not accidentally executed. +- [ ] Screenshots are generated. +- [ ] README explains setup, model choice, demo flow, and limitations. +- [ ] Space deployment is verified or blocker is documented. diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000000000000000000000000000000000000..a531d16fbbaa30ebbbffe2d017790368b71b5fef --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,245 @@ +# How To Use The Project + +## 1. Install Python + +Install Python 3.11 or newer. On Windows, make sure "Add Python to PATH" is enabled. + +Verify: + +```powershell +python --version +``` + +Before Python is available, you can still verify the repository structure: + +```powershell +.\scripts\verify_structure.ps1 +``` + +In this workspace, the direct WindowsApps interpreter worked even though `python` was not on PATH: + +```powershell +& "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" --version +``` + +## 2. Create A Virtual Environment + +```powershell +python -m venv .venv +.venv\Scripts\Activate.ps1 +``` + +If `python` is not on PATH, use the WindowsApps interpreter: + +```powershell +& "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" -m venv .venv +``` + +## 3. Install Dependencies + +```powershell +python -m pip install -r requirements.txt +``` + +For tests and quality checks: + +```powershell +python -m pip install -r requirements-dev.txt +``` + +## 4. Run The App + +```powershell +python app.py +``` + +## 5. Run Tests + +```powershell +.\scripts\run_tests.ps1 +``` + +## 6. Run Quality Checks + +```powershell +.\scripts\run_quality.ps1 +``` + +Open the local Gradio URL printed in the terminal, usually: + +```text +http://127.0.0.1:7860 +``` + +## 7. Current App Behavior + +Local development can still use deterministic helper services for tests, but deployed Space mode +uses `WORKBENCH_DEPLOYMENT=space`, hides placeholder backends, and requires real backend +configuration for model calls. The app still does not download model weights on startup. + +Available tabs. User-triggered tab actions show Gradio progress while callbacks run, and the app +uses compact responsive styling for narrow screens: + +- Chat - llama.cpp, llama-cpp-python, Ollama, OpenAI-compatible/LM Studio, SGLang, or Transformers text inference with tab status/errors. +- Vision - llama.cpp, llama-cpp-python, Ollama, or Transformers MiniCPM image + prompt inference with tab status/errors. +- Dataset - local CSV/JSONL/NDJSON preview, optional Hugging Face dataset preview, stats, and tab status/errors. +- Train - LoRA dry-run training plan plus local base-vs-tuned exact-match evaluation. +- Export - GGUF download/conversion/quantization planning, exported-file listing, and existing-file downloads. +- Field Notes - saves human corrections to CSV, imports uncertain OCR predictions, captures media paths/training flags, exports corrected JSONL, corrected OCR JSONL, and local HF Dataset files. +- Traces - local event preview, JSONL trace rows, tracking status, and trace export. +- Agent - local non-autonomous research-plan-implement-verify trace mode. +- Status - shows configured models, backend metadata, local llama.cpp setup, LM Studio/OpenAI-compatible setup, SGLang command/check/stop planning, and Ollama list/pull planning. + +Ollama is optional and is not installed automatically. Install and start Ollama yourself, then +pull a compatible model explicitly before selecting the Ollama backend in the app. The Status tab +can list models from a running local Ollama server and prepare an explicit +`ollama pull ` command. It shows the command only; it does not run downloads for you. + +llama.cpp is the preferred hackathon backend path. Install llama.cpp separately, open the Status +tab, pick or type an explicit GGUF path, optionally pick an mmproj GGUF for vision, then click +`Prepare local model config`. The app writes `data/local_backends.yaml`, shows the +`llama-server` command, and still does not download or load model weights on startup. +Start `llama-server` yourself with that command, then select the `llama.cpp` backend in the app. +`llama-cpp-python` is also available as an optional backend when the package is installed and a +local GGUF path is configured. + +LM Studio or another local OpenAI-compatible server is optional and is not a cloud API path. Start +the local server yourself, open the Status tab, set the server URL such as +`http://127.0.0.1:1234`, optionally enter the exact served model name shown by LM Studio, and +click `Save OpenAI-compatible config`. `Check server` calls `/v1/models`; selecting +`openai-compatible` in the Chat tab posts to `/v1/chat/completions` only after you submit a prompt. +No model weights are downloaded or loaded on app startup. + +The Transformers text backend is optional. It requires installing `transformers` and a compatible +PyTorch build for your CPU/GPU, and it may download model weights when you explicitly select it and +run a prompt. It is not installed automatically and is not used on startup. + +The Transformers vision backend is optional for MiniCPM-V models. It uses `AutoProcessor` and +`AutoModelForImageTextToText` only after you select `transformers` in the Vision tab and submit an +image. The current app maps the thinking toggle into the prompt template and documents video as a +future frame-sampling path; it does not download or load vision weights on startup. + +SGLang is optional. The Status tab can prepare a local `python -m sglang.launch_server` command, +check `/health`, request `/shutdown`, and the Chat tab can use the `sglang` backend against +`/v1/chat/completions` after you start the server yourself. The app does not start SGLang on +startup. + +Dataset preview supports local `.csv`, `.jsonl`, and `.ndjson` files, split names for optional +Hugging Face dataset preview, and basic local statistics. Field Notes can export corrected +training rows to `data/field_notes.jsonl` and local HF Dataset-style files to +`data/hf_field_notes/`. + +The OCR correction hook expects a local `.csv`, `.jsonl`, or `.ndjson` prediction file with fields +such as `source_path`, `text`, and `confidence`. The Field Notes tab can preview rows at or below a +confidence threshold, import those uncertain rows as correction tasks, and export corrected OCR rows +to `data/ocr_corrections.jsonl`. This is the local OCR -> Field Notes -> Training path: import +uncertain predictions, correct them in Field Notes, then export corrected JSONL/HF Dataset files for +future fine-tuning or evaluation. + +Local MCP-style tools live in `mcp_tools/tools.py`. The selected MCP path is Gradio native MCP, +enabled by `launch(mcp_server=True)`. The documented endpoint path is `/gradio_api/mcp/sse`. +`mcp_tools/bridge.py` exposes a local manifest and invocation helper for dataset stats, optional +HF dataset preview, safe arithmetic, model inference, and non-executing VINDEX call planning. Full +external MCP client verification still depends on launching the app and connecting a client. + +The VINDEX helper is intentionally planning-only. It validates the PRD method names, reports +whether the local `vindex` package or `http://127.0.0.1:8765/health` endpoint is available, and +caps risky edit parameters. It does not edit model weights. + +The Agent tab drafts local research-plan-implement-verify traces and paper-to-code traces, stores +them in `data/agent_traces.jsonl`, and can export JSONL or local HF Dataset-style trace files. It +includes safety gates and does not run shell commands, commit, push, deploy, download models, or +call external services. Agent traces can later become persona/source material for the Desk-Pet +extension, but no Desk-Pet runtime is implemented yet. + +The Export tab is a planning surface. It shows explicit `huggingface-cli`, +`convert_hf_to_gguf.py`, and `llama-quantize` commands for the selected model and quantization, +plus a list of files already present in the export directory. Existing files are exposed through +the download output. It does not run those commands yet. + +The Train tab does not start LoRA training yet. It builds a dry-run plan from +`config/training.yaml`, validates the dataset path, shows the planned checkpoint output directory, +documents hardware expectations, and builds a non-executing LoRA trainer request with PEFT/TRL +dependency status. It can also run a local evaluation by comparing newline-separated base and tuned +responses against the built-in prompt cases. It reports exact match, can calculate perplexity from +optional negative log likelihood values, shows a qualitative table, and appends tuned results to +`data/eval_results.jsonl`. + +The vLLM tab is a local serving planner. It prepares an explicit `vllm serve ...` command, checks +`/health`, fetches `/metrics`, logs parsed benchmark metrics through local tracking, and points the +chat client at `/v1/chat/completions` only after you start vLLM yourself. It does not install vLLM +or start a server on app startup. + +Tracing writes local events to `data/traces.jsonl` by default. The Traces tab can show recent app +events, show JSONL trace rows, report whether optional Trackio is available, and export local +traces to `exports/traces.jsonl`. Remote Trackio/HF sync still requires credentials and setup. + +## 8. Hugging Face Space Deployment + +The repo includes Hugging Face Space metadata in `README.md` and a local planning helper: + +```powershell +.venv\Scripts\python.exe scripts\plan_hf_space.py --user +``` + +The helper validates required files and prints the manual commands for login, Space creation, +remote setup, and push. It does not login, create a repo, push, or store tokens. + +Required Space files: + +- `app.py` +- `requirements.txt` +- `README.md` +- `config/models.yaml` +- `config/training.yaml` + +Target Spaces: + +- Workbench: `https://huggingface.co/spaces/build-small-hackathon/workbench` +- Plant Identification Tool: `https://huggingface.co/spaces/build-small-hackathon/plant_identification_tool` + +Use `hf auth login` with a freshly generated token. The app does not download model weights on +startup; model downloads happen only through explicit backend actions such as `ollama pull`, +`hf download`, or selecting a real Transformers backend and running a prompt. Upgrade Space +hardware if real OpenBMB text or vision inference cannot run on the default CPU tier. + +Trackio/HF sync path: local traces are written to JSONL first, then optional Trackio can be enabled +in `config/training.yaml` after package availability and credentials are ready. + +Synthetic data and reward evaluation helpers are local Python utilities. `datasets/synthetic.py` +can generate, validate, filter, augment, and export JSONL examples. `training/reward_eval.py` can +score supplied responses, select best-of-N candidates, create DPO chosen/rejected pairs, and compare +base responses against LoRA responses. They do not load reward models or call external services. + +## 9. How To Work With Codex + +Useful prompts: + +```text +Read docs/TASKS.md and implement the next unchecked MVP task. +``` + +```text +Read docs/IMPLEMENTATION_STATUS.md and tell me what is blocked. +``` + +```text +Wire the next backend, but keep automatic model downloads disabled on startup. +``` + +```text +Update the docs after the change and mark the matching checklist item. +``` + +```text +If this failed, add or update a test that catches it, then fix the code. +``` + +## 10. What To Avoid + +- Do not start with the whole PRD at once. +- Do not download huge models automatically. +- Do not mark tasks done before running or documenting the blocker. +- Do not add features without tests. +- Do not add broad coverage escapes such as pragma no cover without a documented reason. +- Do not push secrets, tokens, model caches, GGUF files, or virtual environments to git. diff --git a/exports/.gitkeep b/exports/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/exports/.gitkeep @@ -0,0 +1 @@ + diff --git a/mcp_tools/__init__.py b/mcp_tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5bbb8e16f6bbaae59d3736309142bd880c2cf9fe --- /dev/null +++ b/mcp_tools/__init__.py @@ -0,0 +1 @@ +"""Local tool functions that can later be exposed through an MCP endpoint.""" diff --git a/mcp_tools/bridge.py b/mcp_tools/bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..c30b495fb47a15bff01c252203ad2c304c349017 --- /dev/null +++ b/mcp_tools/bridge.py @@ -0,0 +1,55 @@ +from dataclasses import asdict, dataclass +from typing import Any, cast + +from mcp_tools.tools import ToolResult, tool_registry + +MCP_PATH = "/gradio_api/mcp/sse" +MCP_MODE = "gradio_native_mcp_server" + + +@dataclass(frozen=True) +class McpToolDefinition: + name: str + description: str + endpoint: str + + def as_dict(self) -> dict[str, str]: + return asdict(self) + + +TOOL_DESCRIPTIONS = { + "dataset_stats": "Return row, column, and non-empty statistics for a local CSV/JSONL file.", + "hf_dataset_preview": ( + "Preview a Hugging Face dataset when optional dependencies are installed." + ), + "safe_calculator": "Evaluate numeric arithmetic expressions only.", + "model_inference": "Run a text prompt through the selected local model backend.", +} + + +def mcp_tool_definitions() -> list[McpToolDefinition]: + return [ + McpToolDefinition( + name=name, + description=TOOL_DESCRIPTIONS.get(name, "Local workbench tool."), + endpoint=f"{MCP_PATH}#{name}", + ) + for name in sorted(tool_registry()) + ] + + +def mcp_manifest() -> dict[str, Any]: + return { + "mode": MCP_MODE, + "path": MCP_PATH, + "tools": [definition.as_dict() for definition in mcp_tool_definitions()], + "served_by": "Gradio launch(mcp_server=True)", + } + + +def invoke_mcp_tool(name: str, payload: dict[str, Any]) -> ToolResult: + registry = tool_registry() + if name not in registry: + raise KeyError(f"Unknown MCP tool: {name}") + tool = registry[name] + return cast(ToolResult, tool(**payload)) diff --git a/mcp_tools/tools.py b/mcp_tools/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..5592efe4cba942cd3639e6bb6d6a7e49b70a246a --- /dev/null +++ b/mcp_tools/tools.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import ast +import operator +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from datasets.loader import dataset_statistics, preview_huggingface_dataset +from mcp_tools.vindex_tool import vindex_dependency_report, vindex_verification_report +from models.model_catalog import load_model_catalog +from models.service_factory import create_text_service + + +@dataclass(frozen=True) +class ToolResult: + """Simple serializable result for local tool calls.""" + + name: str + payload: dict[str, Any] + + +def dataset_stats_tool(path: str) -> ToolResult: + return ToolResult("dataset_stats", dataset_statistics(path).as_dict()) + + +def hf_dataset_preview_tool(dataset_id: str, split: str = "train") -> ToolResult: + preview = preview_huggingface_dataset(dataset_id, split) + return ToolResult( + "hf_dataset_preview", + { + "dataset_id": preview.dataset_id, + "split": preview.split, + "rows": preview.rows, + "columns": preview.columns, + "samples": preview.samples, + "status": preview.status, + }, + ) + + +def safe_calculator_tool(expression: str) -> ToolResult: + value = _safe_eval(expression) + return ToolResult("safe_calculator", {"expression": expression, "value": value}) + + +def model_inference_tool( + prompt: str, + model_id: str = "minicpm5_1b", + backend: str = "placeholder", + system_prompt: str = "", +) -> ToolResult: + catalog = load_model_catalog("config/models.yaml") + service = create_text_service(catalog[model_id], backend) + response = service.chat(system_prompt, prompt) + return ToolResult( + "model_inference", + { + "model_id": model_id, + "backend": backend, + "response": response, + }, + ) + + +def vindex_status_tool() -> ToolResult: + return ToolResult("vindex_status", vindex_dependency_report()) + + +def vindex_plan_tool(method: str, payload: dict[str, Any] | None = None) -> ToolResult: + return ToolResult( + "vindex_plan", + vindex_verification_report(method=method, payload=payload or {}), + ) + + +def tool_registry() -> dict[str, Any]: + return { + "dataset_stats": dataset_stats_tool, + "hf_dataset_preview": hf_dataset_preview_tool, + "safe_calculator": safe_calculator_tool, + "model_inference": model_inference_tool, + "vindex_status": vindex_status_tool, + "vindex_plan": vindex_plan_tool, + } + + +def _safe_eval(expression: str) -> float: + operators: dict[type[ast.operator] | type[ast.unaryop], Callable[..., float]] = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.Pow: operator.pow, + ast.USub: operator.neg, + } + + def eval_node(node: ast.AST) -> float: + if isinstance(node, ast.Expression): + return eval_node(node.body) + if isinstance(node, ast.Constant) and isinstance(node.value, int | float): + return float(node.value) + if isinstance(node, ast.BinOp) and type(node.op) in operators: + return float(operators[type(node.op)](eval_node(node.left), eval_node(node.right))) + if isinstance(node, ast.UnaryOp) and type(node.op) in operators: + return float(operators[type(node.op)](eval_node(node.operand))) + raise ValueError("Only numeric arithmetic expressions are allowed.") + + return eval_node(ast.parse(expression, mode="eval")) diff --git a/mcp_tools/vindex_tool.py b/mcp_tools/vindex_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..df73f4b340d16c27e6db8e10bb2be2695ebefc22 --- /dev/null +++ b/mcp_tools/vindex_tool.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import importlib.util +from collections.abc import Callable +from dataclasses import asdict, dataclass +from typing import Any + +import requests + +VINDEX_BASE_URL = "http://127.0.0.1:8765" +VINDEX_METHODS = { + "logit_lens": "/logit_lens", + "slot_neighbors": "/slot_neighbors", + "layer_contribution": "/layer_contribution", + "transition_spectrum": "/transition_spectrum", + "calibrated_edit": "/calibrated_edit", + "derive_scale": "/derive_scale", + "star_spread": "/star_spread", + "protect_relations": "/protect_relations", +} + + +@dataclass(frozen=True) +class VINDEXCallPlan: + method: str + endpoint: str + payload: dict[str, Any] + execute: bool + safety_notes: list[str] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def vindex_dependency_report( + base_url: str = VINDEX_BASE_URL, + get_func: Callable[..., requests.Response] = requests.get, +) -> dict[str, Any]: + package_available = importlib.util.find_spec("vindex") is not None + server = _server_status(base_url, get_func) + return { + "package_available": package_available, + "server_base_url": base_url.rstrip("/"), + "server_reachable": server["reachable"], + "server_detail": server["detail"], + "supported_methods": sorted(VINDEX_METHODS), + "dependency": "Install or run VINDEX locally; no cloud endpoint is configured.", + } + + +def build_vindex_call_plan( + method: str, + payload: dict[str, Any], + base_url: str = VINDEX_BASE_URL, +) -> VINDEXCallPlan: + if method not in VINDEX_METHODS: + raise ValueError(f"Unsupported VINDEX method: {method}") + + safe_payload = _safe_payload(method, payload) + return VINDEXCallPlan( + method=method, + endpoint=f"{base_url.rstrip('/')}{VINDEX_METHODS[method]}", + payload=safe_payload, + execute=False, + safety_notes=_safety_notes(method, payload, safe_payload), + ) + + +def vindex_verification_report( + method: str = "logit_lens", + payload: dict[str, Any] | None = None, + base_url: str = VINDEX_BASE_URL, + get_func: Callable[..., requests.Response] = requests.get, +) -> dict[str, Any]: + call_plan = build_vindex_call_plan(method, payload or {"model_id": "", "text": ""}, base_url) + return { + "dependency_report": vindex_dependency_report(base_url, get_func), + "call_plan": call_plan.to_dict(), + "ready_for_execution": False, + "reason": ( + "The workbench records a safe VINDEX boundary first. Execute only after the local " + "VINDEX package/server, model loading, and edit safety checks are verified." + ), + } + + +def _server_status( + base_url: str, + get_func: Callable[..., requests.Response], +) -> dict[str, object]: + url = base_url.rstrip("/") + try: + response = get_func(f"{url}/health", timeout=2) + except requests.RequestException as exc: + return {"reachable": False, "detail": f"VINDEX server not reachable: {exc}"} + if response.ok: + return {"reachable": True, "detail": "VINDEX health endpoint is reachable."} + return { + "reachable": False, + "detail": f"VINDEX health endpoint responded with HTTP {response.status_code}.", + } + + +def _safe_payload(method: str, payload: dict[str, Any]) -> dict[str, Any]: + safe_payload = dict(payload) + if method == "star_spread": + requested = int(safe_payload.get("n_neighbors", 5)) + safe_payload["n_neighbors"] = min(requested, 5) + if method == "calibrated_edit": + requested_window = int(safe_payload.get("causal_window", 3)) + safe_payload["causal_window"] = min(requested_window, 3) + return safe_payload + + +def _safety_notes( + method: str, + original_payload: dict[str, Any], + safe_payload: dict[str, Any], +) -> list[str]: + notes = [ + "Non-executing plan only; no model weights are edited by this workbench helper.", + "Run VINDEX locally and verify protected relations before any calibrated edit.", + ] + if method == "star_spread" and original_payload != safe_payload: + notes.append("n_neighbors was capped at 5 to avoid over-editing related slots.") + if method == "calibrated_edit" and original_payload != safe_payload: + notes.append("causal_window was capped at 3 layers around the phase layer.") + return notes diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a3eee49f031550a5c8156b35e4be9d0d0e3334d4 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1 @@ +"""Model catalog and inference service helpers.""" diff --git a/models/base.py b/models/base.py new file mode 100644 index 0000000000000000000000000000000000000000..5146124712a351e15564bcb64d0e47269d31dfe0 --- /dev/null +++ b/models/base.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from models.model_catalog import ModelInfo + + +@dataclass(frozen=True) +class BackendStatus: + """Current availability of an inference backend.""" + + name: str + available: bool + detail: str + + +class TextModelService(Protocol): + """Text chat service contract.""" + + model: ModelInfo + + def chat(self, system_prompt: str, user_prompt: str) -> str: + """Return a text response.""" + + +class VisionModelService(Protocol): + """Vision chat service contract.""" + + model: ModelInfo + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + """Return a vision-language response.""" diff --git a/models/hf_components.py b/models/hf_components.py new file mode 100644 index 0000000000000000000000000000000000000000..d0b143749b4937b054695eaaa0dd2bb4c8de6a0a --- /dev/null +++ b/models/hf_components.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from importlib import import_module + +from models.model_catalog import ModelInfo + + +def load_tokenizer_and_causal_lm( + model: ModelInfo, + trust_remote_code: bool, + device_map: str, + torch_dtype: str, +): + transformers_module = import_module("transformers") + tokenizer = transformers_module.AutoTokenizer.from_pretrained( + model.hf_id, + trust_remote_code=trust_remote_code, + ) + model_instance = transformers_module.AutoModelForCausalLM.from_pretrained( + model.hf_id, + trust_remote_code=trust_remote_code, + device_map=device_map, + torch_dtype=torch_dtype, + ) + return model_instance, tokenizer + + +def load_processor_and_image_text_model( + model: ModelInfo, + trust_remote_code: bool, + device_map: str, + torch_dtype: str, +): + transformers_module = import_module("transformers") + processor = transformers_module.AutoProcessor.from_pretrained( + model.hf_id, + trust_remote_code=trust_remote_code, + ) + model_instance = transformers_module.AutoModelForImageTextToText.from_pretrained( + model.hf_id, + trust_remote_code=trust_remote_code, + device_map=device_map, + torch_dtype=torch_dtype, + ) + return model_instance, processor diff --git a/models/http_chat.py b/models/http_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..45411b482e44670e2e27473c927ad548b3bf2e32 --- /dev/null +++ b/models/http_chat.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, cast + +import requests + +from models.response_parsing import chat_completion_payload + + +def post_json( + post_func: Callable[..., requests.Response], + url: str, + payload: dict[str, Any], + timeout: float, + headers: dict[str, str] | None = None, +) -> dict[str, Any]: + response = post_func( + url, + json=payload, + headers=headers, + timeout=timeout, + ) + response.raise_for_status() + return cast(dict[str, Any], response.json()) + + +def post_chat_completion( + post_func: Callable[..., requests.Response], + url: str, + model: str, + system_prompt: str, + user_prompt: str, + temperature: float, + max_tokens: int, + timeout: float, + headers: dict[str, str] | None = None, +) -> dict[str, Any]: + return post_json( + post_func, + url, + chat_completion_payload(model, system_prompt, user_prompt, temperature, max_tokens), + timeout, + headers, + ) diff --git a/models/llama_cpp_python_service.py b/models/llama_cpp_python_service.py new file mode 100644 index 0000000000000000000000000000000000000000..8cf18d15eb84e939da7d9a053f8a48d4c2f87eb2 --- /dev/null +++ b/models/llama_cpp_python_service.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass +from importlib import import_module +from pathlib import Path +from typing import Any + +from models.base import BackendStatus +from models.model_catalog import ModelInfo +from models.response_parsing import extract_chat_response + + +@dataclass(frozen=True) +class LlamaCppPythonConfig: + """Runtime configuration for llama-cpp-python.""" + + model_path: str = "" + n_ctx: int = 4096 + n_gpu_layers: int = 0 + temperature: float = 0.7 + max_tokens: int = 512 + + +class LlamaCppPythonService: + """Direct llama-cpp-python GGUF inference service.""" + + def __init__( + self, + model: ModelInfo, + config: LlamaCppPythonConfig | None = None, + ) -> None: + self.model = model + self.config = config or LlamaCppPythonConfig() + + @staticmethod + def status(model_path: str = "") -> BackendStatus: + if importlib.util.find_spec("llama_cpp") is None: + return BackendStatus( + "llama-cpp-python", + False, + "Python package llama-cpp-python is not installed in the current environment.", + ) + if not model_path: + return BackendStatus( + "llama-cpp-python", + False, + "llama-cpp-python is installed, but no GGUF model path is configured.", + ) + if not Path(model_path).exists(): + return BackendStatus( + "llama-cpp-python", + False, + f"Configured GGUF model was not found: {model_path}", + ) + return BackendStatus("llama-cpp-python", True, "llama-cpp-python is ready.") + + def chat(self, system_prompt: str, user_prompt: str) -> str: + status = self.status(self.config.model_path) + if not status.available: + return ( + "[llama-cpp-python unavailable]\n\n" + f"{status.detail}\n\n" + "Install llama-cpp-python and configure a local GGUF path before retrying." + ) + + llama = self._load_llama() + response = llama.create_chat_completion( + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + temperature=self.config.temperature, + max_tokens=self.config.max_tokens, + ) + return self._extract_response(response) + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + del image + if has_image: + return ( + "[llama-cpp-python vision note]\n\n" + "Direct multimodal llama-cpp-python support requires model-specific mmproj " + "wiring and image serialization. Use llama-server for the current vision path." + ) + return self.chat("", prompt) + + def _load_llama(self): + llama_module = import_module("llama_cpp") + llama_class = llama_module.Llama + + return llama_class( + model_path=self.config.model_path, + n_ctx=self.config.n_ctx, + n_gpu_layers=self.config.n_gpu_layers, + verbose=False, + ) + + @staticmethod + def _extract_response(data: dict[str, Any]) -> str: + return extract_chat_response(data) diff --git a/models/llama_cpp_service.py b/models/llama_cpp_service.py new file mode 100644 index 0000000000000000000000000000000000000000..234ec1863e68ac56eef92777001a4f5d77986b99 --- /dev/null +++ b/models/llama_cpp_service.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import shutil +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import requests + +from models.base import BackendStatus +from models.model_catalog import ModelInfo +from models.response_parsing import extract_chat_response + + +@dataclass(frozen=True) +class LlamaCppConfig: + """Runtime configuration for a local llama.cpp server.""" + + server_url: str = "http://127.0.0.1:8080" + server_path: str = "" + model_path: str = "" + mmproj_path: str = "" + + +class LlamaCppService: + """llama.cpp HTTP client for local GGUF inference.""" + + def __init__( + self, + model: ModelInfo, + config: LlamaCppConfig | None = None, + timeout_seconds: float = 60, + ) -> None: + self.model = model + self.config = config or LlamaCppConfig() + self.timeout_seconds = timeout_seconds + + @staticmethod + def status( + which_func: Callable[[str], str | None] = shutil.which, + get_func: Callable[..., requests.Response] = requests.get, + server_url: str = "http://127.0.0.1:8080", + server_path: str = "", + ) -> BackendStatus: + executable = server_path or "llama-server" + if server_path: + if not Path(server_path).exists(): + return BackendStatus( + "llama.cpp", + False, + f"Configured llama-server was not found: {server_path}", + ) + elif which_func(executable) is None: + return BackendStatus("llama.cpp", False, "llama-server was not found on PATH.") + + try: + response = get_func(f"{server_url}/health", timeout=2) + except requests.RequestException as exc: + return BackendStatus( + "llama.cpp", + False, + f"llama-server is installed but not reachable: {exc}", + ) + + if response.ok: + return BackendStatus("llama.cpp", True, "llama-server is installed and reachable.") + return BackendStatus( + "llama.cpp", + False, + f"llama-server responded with HTTP {response.status_code}.", + ) + + def launch_command(self) -> list[str]: + if not self.config.model_path: + return [] + + command = [self.config.server_path or "llama-server", "-m", self.config.model_path] + if self.config.mmproj_path: + command.extend(["--mmproj", self.config.mmproj_path]) + return command + + def chat(self, system_prompt: str, user_prompt: str) -> str: + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + return self._post_chat(messages) + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + del image + if has_image: + return ( + "[llama.cpp vision note]\n\n" + "Image upload requires a running llama-server with an mmproj file. " + "The current scaffold validates the server path but does not yet serialize " + "Gradio images into llama.cpp multimodal payloads." + ) + return self._post_chat([{"role": "user", "content": prompt}]) + + def _post_chat(self, messages: list[dict[str, str]]) -> str: + status = self.status( + server_url=self.config.server_url, + server_path=self.config.server_path, + ) + if not status.available: + return ( + "[llama.cpp unavailable]\n\n" + f"{status.detail}\n\n" + "Install llama.cpp, start llama-server with an explicit GGUF model, " + "then retry." + ) + + try: + response = requests.post( + f"{self.config.server_url}/v1/chat/completions", + json={ + "messages": messages, + "temperature": 0.7, + "max_tokens": 512, + }, + timeout=self.timeout_seconds, + ) + response.raise_for_status() + except requests.RequestException as exc: + return f"[llama.cpp request failed]\n\n{exc}" + + return self._extract_response(dict(response.json())) + + @staticmethod + def _extract_response(data: dict[str, Any]) -> str: + return extract_chat_response(data) + + +def local_file_status(path: str) -> str: + if not path: + return "not configured" + return "found" if Path(path).exists() else "missing" diff --git a/models/local_backend_config.py b/models/local_backend_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d192d74cef7d38cd4e6cb62659ccc144bdedfbae --- /dev/null +++ b/models/local_backend_config.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import yaml + +LOCAL_BACKEND_CONFIG_PATH = Path("data/local_backends.yaml") + + +@dataclass(frozen=True) +class LocalBackendConfig: + """User-local backend settings stored outside git-tracked config.""" + + llama_cpp_server_url: str = "http://127.0.0.1:8080" + llama_server_path: str = "" + openai_compatible_base_url: str = "http://127.0.0.1:1234" + openai_compatible_model_name: str = "" + gguf_path: str = "" + mmproj_path: str = "" + n_ctx: int = 4096 + n_gpu_layers: int = 0 + + def file_status_rows(self) -> list[list[str]]: + return [ + ["llama-server", self.llama_server_path, _local_file_status(self.llama_server_path)], + ["GGUF model", self.gguf_path, _local_file_status(self.gguf_path)], + ["mmproj", self.mmproj_path, _local_file_status(self.mmproj_path)], + ] + + +def load_local_backend_config( + path: str | Path = LOCAL_BACKEND_CONFIG_PATH, +) -> LocalBackendConfig: + config_path = Path(path) + if not config_path.exists(): + return LocalBackendConfig() + + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + llama_cpp = raw.get("llama_cpp", {}) + openai_compatible = raw.get("openai_compatible", {}) + return LocalBackendConfig( + llama_cpp_server_url=str( + llama_cpp.get("server_url", LocalBackendConfig.llama_cpp_server_url) + ), + llama_server_path=str(llama_cpp.get("llama_server_path", "")), + openai_compatible_base_url=str( + openai_compatible.get( + "base_url", + LocalBackendConfig.openai_compatible_base_url, + ) + ), + openai_compatible_model_name=str(openai_compatible.get("model_name", "")), + gguf_path=str(llama_cpp.get("gguf_path", "")), + mmproj_path=str(llama_cpp.get("mmproj_path", "")), + n_ctx=int(llama_cpp.get("n_ctx", LocalBackendConfig.n_ctx)), + n_gpu_layers=int(llama_cpp.get("n_gpu_layers", LocalBackendConfig.n_gpu_layers)), + ) + + +def save_local_backend_config( + config: LocalBackendConfig, + path: str | Path = LOCAL_BACKEND_CONFIG_PATH, +) -> Path: + config_path = Path(path) + config_path.parent.mkdir(parents=True, exist_ok=True) + data: dict[str, dict[str, Any]] = { + "llama_cpp": { + "server_url": config.llama_cpp_server_url, + "llama_server_path": config.llama_server_path, + "gguf_path": config.gguf_path, + "mmproj_path": config.mmproj_path, + "n_ctx": config.n_ctx, + "n_gpu_layers": config.n_gpu_layers, + }, + "openai_compatible": { + "base_url": config.openai_compatible_base_url, + "model_name": config.openai_compatible_model_name, + }, + } + config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + return config_path + + +def build_llama_server_command(config: LocalBackendConfig) -> list[str]: + if not config.gguf_path: + return [] + + executable = config.llama_server_path or "llama-server" + command = [executable, "-m", config.gguf_path] + if config.mmproj_path: + command.extend(["--mmproj", config.mmproj_path]) + return command + + +def local_backend_summary(config: LocalBackendConfig) -> dict[str, Any]: + command = build_llama_server_command(config) + return { + **asdict(config), + "llama_server_command": " ".join(command) if command else "", + "files": config.file_status_rows(), + "startup_downloads": False, + "auto_model_load": False, + } + + +def _local_file_status(path: str) -> str: + if not path: + return "not configured" + return "found" if Path(path).exists() else "missing" diff --git a/models/minicpm_vision.py b/models/minicpm_vision.py new file mode 100644 index 0000000000000000000000000000000000000000..c4bcc6f54116ee62dda931dce9b5d341d514ac58 --- /dev/null +++ b/models/minicpm_vision.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass +from typing import Any, cast + +from models.base import BackendStatus +from models.hf_components import load_processor_and_image_text_model +from models.model_catalog import ModelInfo + + +@dataclass(frozen=True) +class MiniCPMVisionConfig: + trust_remote_code: bool = True + device_map: str = "auto" + torch_dtype: str = "auto" + max_new_tokens: int = 256 + temperature: float = 0.2 + do_sample: bool = False + + +class MiniCPMVisionService: + """Optional MiniCPM vision backend using Transformers image-text-to-text APIs.""" + + def __init__( + self, + model: ModelInfo, + config: MiniCPMVisionConfig | None = None, + ) -> None: + self.model = model + self.config = config or MiniCPMVisionConfig( + trust_remote_code=model.trust_remote_code or True + ) + self._model = None + self._processor = None + + @staticmethod + def status() -> BackendStatus: + if importlib.util.find_spec("transformers") is None: + return BackendStatus( + "transformers-vision", + False, + "Python package transformers is not installed in the current environment.", + ) + return BackendStatus( + "transformers-vision", + True, + "Transformers package is installed; vision model loads only when selected.", + ) + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + if not has_image: + return "[MiniCPM vision unavailable]\n\nAdd an image before running this backend." + + status = self.status() + if not status.available: + return ( + "[MiniCPM vision unavailable]\n\n" + f"{status.detail}\n\n" + "Install transformers/torch and select this backend only when local hardware " + "can load the chosen vision model." + ) + + model, processor = self._load_components() + messages = self.format_messages(prompt, image, thinking=self.model.thinking_mode) + inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ) + outputs = model.generate(**inputs, **self.generation_kwargs()) + decoded = cast(str, processor.decode(outputs[0], skip_special_tokens=True)) + return decoded.strip() + + def generation_kwargs(self) -> dict[str, Any]: + return { + "max_new_tokens": self.config.max_new_tokens, + "temperature": self.config.temperature, + "do_sample": self.config.do_sample, + } + + @staticmethod + def format_messages(prompt: str, image, thinking: bool = False) -> list[dict[str, Any]]: + text = prompt.strip() or "Describe the image." + if thinking: + text = f"{text}\nThink carefully before answering." + return [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": text}, + ], + } + ] + + @staticmethod + def video_support_plan() -> dict[str, object]: + return { + "implemented": False, + "reason": "Current Gradio tab accepts one image. Video requires frame sampling first.", + "next_steps": [ + "Add video upload input.", + "Sample frames locally without uploading media.", + "Pass frame list through the model-specific processor template.", + "Add tests with synthetic frame placeholders before enabling execution.", + ], + } + + def _load_components(self): + if self._model is not None and self._processor is not None: + return self._model, self._processor + + self._model, self._processor = load_processor_and_image_text_model( + self.model, + self.config.trust_remote_code, + self.config.device_map, + self.config.torch_dtype, + ) + return self._model, self._processor diff --git a/models/model_catalog.py b/models/model_catalog.py new file mode 100644 index 0000000000000000000000000000000000000000..efae53686dc3b84ba650818714106ae77c445b92 --- /dev/null +++ b/models/model_catalog.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass(frozen=True) +class ModelInfo: + """Configured model metadata.""" + + config_id: str + hf_id: str + display_name: str + type: str + parameters_b: float + backend: str + context_length: int + notes: str + thinking_mode: bool + gguf: dict[str, Any] + backend_capabilities: dict[str, list[str]] + trust_remote_code: bool + + +def load_model_catalog(path: str) -> dict[str, ModelInfo]: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + catalog = {} + for config_id, cfg in raw.get("models", {}).items(): + catalog[config_id] = ModelInfo( + config_id=config_id, + hf_id=cfg["hf_id"], + display_name=cfg.get("display_name", config_id), + type=cfg.get("type", "text"), + parameters_b=float(cfg.get("parameters_b", 0)), + backend=cfg.get("backend", "placeholder"), + context_length=int(cfg.get("context_length", 0)), + notes=cfg.get("notes", ""), + thinking_mode=bool(cfg.get("thinking_mode", False)), + gguf=cfg.get("gguf", {}), + backend_capabilities={ + backend: list(capabilities) + for backend, capabilities in cfg.get("backend_capabilities", {}).items() + }, + trust_remote_code=bool(cfg.get("trust_remote_code", False)), + ) + return catalog + + +def model_choices(catalog: dict[str, ModelInfo], model_type: str | None = None) -> list[str]: + choices = [] + for model in catalog.values(): + if model_type and model.type != model_type: + continue + choices.append(model.config_id) + return choices + + +def model_summary(model: ModelInfo) -> dict[str, Any]: + return { + "config_id": model.config_id, + "hf_id": model.hf_id, + "display_name": model.display_name, + "type": model.type, + "parameters_b": model.parameters_b, + "backend": model.backend, + "context_length": model.context_length, + "thinking_mode": model.thinking_mode, + "gguf": model.gguf, + "backend_capabilities": model.backend_capabilities, + "trust_remote_code": model.trust_remote_code, + "notes": model.notes, + } + + +def validate_catalog(catalog: dict[str, ModelInfo], max_parameters_b: float = 32) -> list[str]: + warnings = [] + for model in catalog.values(): + if model.parameters_b > max_parameters_b: + warnings.append( + f"{model.config_id} has {model.parameters_b}B parameters, " + f"above the {max_parameters_b}B limit." + ) + if model.backend == "placeholder": + warnings.append( + f"{model.config_id} uses placeholder backend; real inference is not wired yet." + ) + if model.backend not in model.backend_capabilities: + warnings.append( + f"{model.config_id} has no capability metadata for backend {model.backend}." + ) + if not model.hf_id: + warnings.append(f"{model.config_id} is missing hf_id.") + return warnings diff --git a/models/ollama_service.py b/models/ollama_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e5a3060a272d8694d4f2fef4b9ae4c96958602ed --- /dev/null +++ b/models/ollama_service.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import base64 +import shutil +from collections.abc import Callable +from io import BytesIO +from typing import Any + +import requests + +from models.base import BackendStatus +from models.model_catalog import ModelInfo + + +class OllamaService: + """Ollama-backed local inference service.""" + + base_url = "http://127.0.0.1:11434" + + def __init__(self, model: ModelInfo, timeout_seconds: float = 60) -> None: + self.model = model + self.timeout_seconds = timeout_seconds + + @staticmethod + def status( + which_func: Callable[[str], str | None] = shutil.which, + get_func: Callable[..., requests.Response] = requests.get, + ) -> BackendStatus: + if which_func("ollama") is None: + return BackendStatus("ollama", False, "Ollama executable was not found on PATH.") + + try: + response = get_func(f"{OllamaService.base_url}/api/tags", timeout=2) + except requests.RequestException as exc: + return BackendStatus("ollama", False, f"Ollama is installed but not reachable: {exc}") + + if response.ok: + return BackendStatus("ollama", True, "Ollama is installed and reachable.") + return BackendStatus( + "ollama", + False, + f"Ollama responded with HTTP {response.status_code}.", + ) + + def chat(self, system_prompt: str, user_prompt: str) -> str: + payload: dict[str, Any] = { + "model": self.model.hf_id, + "stream": False, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + } + data = self._post_chat(payload) + return str(data.get("message", {}).get("content", "")) + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + message: dict[str, Any] = {"role": "user", "content": prompt} + if has_image and image is not None: + message["images"] = [self._encode_image(image)] + + payload = { + "model": self.model.hf_id, + "stream": False, + "messages": [message], + } + data = self._post_chat(payload) + return str(data.get("message", {}).get("content", "")) + + @classmethod + def list_local_models( + cls, + get_func: Callable[..., requests.Response] = requests.get, + ) -> list[str]: + try: + response = get_func(f"{cls.base_url}/api/tags", timeout=2) + response.raise_for_status() + except requests.RequestException: + return [] + + data = response.json() + return [str(model.get("name", "")) for model in data.get("models", []) if model.get("name")] + + @staticmethod + def pull_command(model_name: str) -> list[str]: + if not model_name.strip(): + return [] + return ["ollama", "pull", model_name.strip()] + + def _post_chat(self, payload: dict[str, Any]) -> dict[str, Any]: + status = self.status() + if not status.available: + return { + "message": { + "content": ( + "[Ollama unavailable]\n\n" + f"{status.detail}\n\n" + "Install/start Ollama and pull the selected model explicitly " + "before retrying." + ) + } + } + + try: + response = requests.post( + f"{self.base_url}/api/chat", + json=payload, + timeout=self.timeout_seconds, + ) + response.raise_for_status() + except requests.RequestException as exc: + return {"message": {"content": f"[Ollama request failed]\n\n{exc}"}} + + return dict(response.json()) + + @staticmethod + def _encode_image(image) -> str: + buffer = BytesIO() + image.save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode("ascii") diff --git a/models/openai_compatible_service.py b/models/openai_compatible_service.py new file mode 100644 index 0000000000000000000000000000000000000000..49bbedf5ed9e3cb01de52f13956d7e788fa2b543 --- /dev/null +++ b/models/openai_compatible_service.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import requests + +from models.base import BackendStatus +from models.model_catalog import ModelInfo +from models.response_parsing import extract_chat_response +from models.server_helpers import request_openai_chat_text + + +@dataclass(frozen=True) +class OpenAICompatibleConfig: + base_url: str = "http://127.0.0.1:1234" + api_key: str = "lm-studio" + model_name: str = "" + temperature: float = 0.7 + max_tokens: int = 512 + timeout_seconds: float = 120 + + @property + def api_base(self) -> str: + return self.base_url.rstrip("/") + + def request_model(self, fallback: str) -> str: + return self.model_name.strip() or fallback + + +class OpenAICompatibleService: + """Local OpenAI-compatible chat client for LM Studio, vLLM, or similar servers.""" + + def __init__( + self, + model: ModelInfo, + config: OpenAICompatibleConfig | None = None, + get_func: Callable[..., requests.Response] = requests.get, + post_func: Callable[..., requests.Response] = requests.post, + ) -> None: + self.model = model + self.config = config or OpenAICompatibleConfig() + self.get_func = get_func + self.post_func = post_func + + @staticmethod + def status( + base_url: str = "http://127.0.0.1:1234", + get_func: Callable[..., requests.Response] = requests.get, + ) -> BackendStatus: + url = base_url.rstrip("/") + try: + response = get_func(f"{url}/v1/models", timeout=2) + response.raise_for_status() + except requests.RequestException as exc: + return BackendStatus( + "openai-compatible", + False, + f"OpenAI-compatible local server is not reachable at {url}: {exc}", + ) + return BackendStatus( + "openai-compatible", + True, + f"OpenAI-compatible local server is reachable at {url}.", + ) + + def chat(self, system_prompt: str, user_prompt: str) -> str: + status = self.status(self.config.api_base, self.get_func) + if not status.available: + return ( + "[OpenAI-compatible backend unavailable]\n\n" + f"{status.detail}\n\n" + "Start LM Studio or another local OpenAI-compatible server before retrying." + ) + + return request_openai_chat_text( + post_func=self.post_func, + base_url=self.config.api_base, + model=self.config.request_model(self.model.hf_id), + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=self.config.temperature, + max_tokens=self.config.max_tokens, + timeout_seconds=self.config.timeout_seconds, + failure_label="OpenAI-compatible", + headers={"Authorization": f"Bearer {self.config.api_key}"}, + ) + + @staticmethod + def _extract_response(data: dict[str, Any]) -> str: + return extract_chat_response(data) diff --git a/models/placeholder_service.py b/models/placeholder_service.py new file mode 100644 index 0000000000000000000000000000000000000000..5da0140feef0fc0de903c7353d6e0e2e58efe2f4 --- /dev/null +++ b/models/placeholder_service.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from models.model_catalog import ModelInfo + + +class PlaceholderModelService: + """Deterministic scaffold service used until a real backend is configured.""" + + def __init__(self, model: ModelInfo) -> None: + self.model = model + + def chat(self, system_prompt: str, user_prompt: str) -> str: + return ( + f"[Placeholder response from {self.model.display_name}]\n\n" + "Real inference is not wired yet. Configure Ollama, llama.cpp, Transformers, " + "or SGLang before using this as a hackathon demo.\n\n" + f"System prompt: {system_prompt or '(none)'}\n" + f"User prompt: {user_prompt}" + ) + + def vision_chat(self, has_image: bool, prompt: str, image=None) -> str: + del image + image_state = "an uploaded image" if has_image else "no image" + return ( + f"[Placeholder vision response from {self.model.display_name}]\n\n" + f"I received {image_state}. Real MiniCPM-V inference is not wired yet.\n\n" + f"Prompt: {prompt}" + ) diff --git a/models/response_parsing.py b/models/response_parsing.py new file mode 100644 index 0000000000000000000000000000000000000000..be819fb3635974422f5298d5bdd66e5c32795c5a --- /dev/null +++ b/models/response_parsing.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import Any + + +def chat_messages(system_prompt: str, user_prompt: str) -> list[dict[str, str]]: + return [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + + +def chat_completion_payload( + model: str, + system_prompt: str, + user_prompt: str, + temperature: float, + max_tokens: int, +) -> dict[str, Any]: + return { + "model": model, + "messages": chat_messages(system_prompt, user_prompt), + "temperature": temperature, + "max_tokens": max_tokens, + } + + +def extract_chat_response(data: dict[str, Any]) -> str: + choices = data.get("choices", []) + if not choices: + return "" + first = choices[0] + if not isinstance(first, dict): + return "" + message = first.get("message", {}) + if isinstance(message, dict) and message.get("content"): + return str(message["content"]) + return str(first.get("text", "")) diff --git a/models/server_helpers.py b/models/server_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..dd29b4afb5e6a059b81ab44251f0d70f1fb6a666 --- /dev/null +++ b/models/server_helpers.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import requests + +from models.http_chat import post_chat_completion +from models.response_parsing import extract_chat_response + + +def host_port_args(host: str, port: int) -> list[str]: + return ["--host", host, "--port", str(port)] + + +def request_openai_chat_completion( + post_func: Callable[..., requests.Response], + base_url: str, + model: str, + system_prompt: str, + user_prompt: str, + temperature: float, + max_tokens: int, + timeout_seconds: float, + headers: dict[str, str] | None = None, +) -> dict[str, Any]: + return post_chat_completion( + post_func=post_func, + url=f"{base_url.rstrip('/')}/v1/chat/completions", + model=model, + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=temperature, + max_tokens=max_tokens, + timeout=timeout_seconds, + headers=headers, + ) + + +def request_openai_chat_text( + post_func: Callable[..., requests.Response], + base_url: str, + model: str, + system_prompt: str, + user_prompt: str, + temperature: float, + max_tokens: int, + timeout_seconds: float, + failure_label: str, + headers: dict[str, str] | None = None, +) -> str: + try: + data = request_openai_chat_completion( + post_func=post_func, + base_url=base_url, + model=model, + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=temperature, + max_tokens=max_tokens, + timeout_seconds=timeout_seconds, + headers=headers, + ) + except requests.RequestException as exc: + return f"[{failure_label} request failed]\n\n{exc}" + return extract_chat_response(data) diff --git a/models/service_factory.py b/models/service_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..96972abd61b05fcd65f509cf46816657b8a8f891 --- /dev/null +++ b/models/service_factory.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from collections.abc import Callable + +from core.deployment import ( + DeploymentPolicy, + current_policy, + ensure_backend_allowed, + filter_backends_for_policy, +) +from core.registry import Registry +from models.base import BackendStatus, TextModelService, VisionModelService +from models.llama_cpp_python_service import LlamaCppPythonConfig, LlamaCppPythonService +from models.llama_cpp_service import LlamaCppConfig, LlamaCppService +from models.local_backend_config import load_local_backend_config +from models.minicpm_vision import MiniCPMVisionService +from models.model_catalog import ModelInfo +from models.ollama_service import OllamaService +from models.openai_compatible_service import OpenAICompatibleConfig, OpenAICompatibleService +from models.placeholder_service import PlaceholderModelService +from models.sglang_runner import SGLangConfig, SGLangService +from models.transformers_text import TransformersTextService + +TextFactory = Callable[[ModelInfo], TextModelService] +VisionFactory = Callable[[ModelInfo], VisionModelService] + +TEXT_SERVICE_REGISTRY: Registry[TextFactory] = Registry() +TEXT_SERVICE_REGISTRY.register("placeholder", PlaceholderModelService) +TEXT_SERVICE_REGISTRY.register("llama.cpp", LlamaCppService) +TEXT_SERVICE_REGISTRY.register("llama-cpp-python", LlamaCppPythonService) +TEXT_SERVICE_REGISTRY.register("ollama", OllamaService) +TEXT_SERVICE_REGISTRY.register("transformers", TransformersTextService) +TEXT_SERVICE_REGISTRY.register("openai-compatible", OpenAICompatibleService) +TEXT_SERVICE_REGISTRY.register("sglang", SGLangService) + +VISION_SERVICE_REGISTRY: Registry[VisionFactory] = Registry() +VISION_SERVICE_REGISTRY.register("placeholder", PlaceholderModelService) +VISION_SERVICE_REGISTRY.register("llama.cpp", LlamaCppService) +VISION_SERVICE_REGISTRY.register("llama-cpp-python", LlamaCppPythonService) +VISION_SERVICE_REGISTRY.register("ollama", OllamaService) +VISION_SERVICE_REGISTRY.register("transformers", MiniCPMVisionService) + +BACKENDS = TEXT_SERVICE_REGISTRY.list() + + +def create_text_service( + model: ModelInfo, + backend: str, + policy: DeploymentPolicy | None = None, +) -> TextModelService: + active_policy = policy or current_policy() + ensure_backend_allowed(backend, active_policy) + if backend == "llama.cpp": + config = load_local_backend_config() + return LlamaCppService( + model, + LlamaCppConfig( + server_url=config.llama_cpp_server_url, + server_path=config.llama_server_path, + model_path=config.gguf_path, + mmproj_path=config.mmproj_path, + ), + ) + if backend == "llama-cpp-python": + config = load_local_backend_config() + return LlamaCppPythonService( + model, + LlamaCppPythonConfig( + model_path=config.gguf_path, + n_ctx=config.n_ctx, + n_gpu_layers=config.n_gpu_layers, + ), + ) + if backend == "openai-compatible": + config = load_local_backend_config() + return OpenAICompatibleService( + model, + OpenAICompatibleConfig( + base_url=config.openai_compatible_base_url, + model_name=config.openai_compatible_model_name, + ), + ) + if backend == "sglang": + return SGLangService(model, SGLangConfig()) + return TEXT_SERVICE_REGISTRY.get(backend)(model) + + +def create_vision_service( + model: ModelInfo, + backend: str, + policy: DeploymentPolicy | None = None, +) -> VisionModelService: + active_policy = policy or current_policy() + ensure_backend_allowed(backend, active_policy) + if backend == "llama.cpp": + config = load_local_backend_config() + return LlamaCppService( + model, + LlamaCppConfig( + server_url=config.llama_cpp_server_url, + server_path=config.llama_server_path, + model_path=config.gguf_path, + mmproj_path=config.mmproj_path, + ), + ) + if backend == "llama-cpp-python": + config = load_local_backend_config() + return LlamaCppPythonService( + model, + LlamaCppPythonConfig( + model_path=config.gguf_path, + n_ctx=config.n_ctx, + n_gpu_layers=config.n_gpu_layers, + ), + ) + return VISION_SERVICE_REGISTRY.get(backend)(model) + + +def backend_statuses(policy: DeploymentPolicy | None = None) -> list[BackendStatus]: + active_policy = policy or current_policy() + statuses = [ + BackendStatus("placeholder", True, "Deterministic placeholder backend is available."), + LlamaCppService.status(), + LlamaCppPythonService.status(), + OllamaService.status(), + TransformersTextService.status(), + OpenAICompatibleService.status(load_local_backend_config().openai_compatible_base_url), + SGLangService.status(), + MiniCPMVisionService.status(), + ] + allowed_backend_names = set(filter_backends_for_policy(BACKENDS, active_policy)) + if active_policy.allow_placeholder_backend: + allowed_backend_names.add("placeholder") + return [ + status + for status in statuses + if status.name in allowed_backend_names or status.name == "transformers-vision" + ] diff --git a/models/sglang_runner.py b/models/sglang_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..06ef757bd32ed259f1f038b037c72c66c62a64d7 --- /dev/null +++ b/models/sglang_runner.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import importlib.util +import shutil +from collections.abc import Callable +from dataclasses import asdict, dataclass +from typing import Any + +import requests + +from models.base import BackendStatus +from models.model_catalog import ModelInfo +from models.server_helpers import host_port_args, request_openai_chat_text + + +@dataclass(frozen=True) +class SGLangConfig: + base_url: str = "http://127.0.0.1:30000" + host: str = "127.0.0.1" + port: int = 30000 + tp_size: int = 1 + tool_parser: str = "minicpm" + temperature: float = 0.7 + max_tokens: int = 512 + timeout_seconds: float = 120 + + +@dataclass(frozen=True) +class SGLangRunPlan: + start_command: list[str] + stop_url: str + health_url: str + chat_url: str + startup_downloads: bool + auto_model_load: bool + notes: list[str] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class SGLangService: + """SGLang OpenAI-compatible client and local server command planner.""" + + def __init__( + self, + model: ModelInfo, + config: SGLangConfig | None = None, + get_func: Callable[..., requests.Response] = requests.get, + post_func: Callable[..., requests.Response] = requests.post, + ) -> None: + self.model = model + self.config = config or SGLangConfig() + self.get_func = get_func + self.post_func = post_func + + @staticmethod + def status( + base_url: str = "http://127.0.0.1:30000", + which_func: Callable[[str], str | None] = shutil.which, + find_spec: Callable[[str], object | None] = importlib.util.find_spec, + get_func: Callable[..., requests.Response] = requests.get, + ) -> BackendStatus: + has_cli = which_func("python") is not None and find_spec("sglang") is not None + try: + response = get_func(f"{base_url.rstrip('/')}/health", timeout=2) + except requests.RequestException as exc: + detail = f"SGLang server is not reachable: {exc}" + if not has_cli: + detail += " Python package sglang is also not installed." + return BackendStatus("sglang", False, detail) + if response.ok: + return BackendStatus("sglang", True, "SGLang server is reachable.") + return BackendStatus("sglang", False, f"SGLang responded with HTTP {response.status_code}.") + + def run_plan(self) -> SGLangRunPlan: + return build_sglang_run_plan(self.model, self.config) + + def chat(self, system_prompt: str, user_prompt: str) -> str: + status = self.status(self.config.base_url, get_func=self.get_func) + if not status.available: + return ( + "[SGLang unavailable]\n\n" + f"{status.detail}\n\n" + "Install SGLang, start the planned local server command, then retry." + ) + + return request_openai_chat_text( + self.post_func, + self.config.base_url, + self.model.hf_id, + system_prompt, + user_prompt, + self.config.temperature, + self.config.max_tokens, + self.config.timeout_seconds, + "SGLang", + ) + + def stop_server(self) -> str: + try: + response = self.post_func( + f"{self.config.base_url.rstrip('/')}/shutdown", + timeout=5, + ) + response.raise_for_status() + except requests.RequestException as exc: + return f"[SGLang stop failed]\n\n{exc}" + return "SGLang shutdown request sent." + + +def build_sglang_run_plan( + model: ModelInfo, + config: SGLangConfig | None = None, +) -> SGLangRunPlan: + cfg = config or SGLangConfig() + base_url = cfg.base_url.rstrip("/") + return SGLangRunPlan( + start_command=[ + "python", + "-m", + "sglang.launch_server", + "--model-path", + model.hf_id, + *host_port_args(cfg.host, cfg.port), + "--tp-size", + str(cfg.tp_size), + "--tool-call-parser", + cfg.tool_parser, + ], + stop_url=f"{base_url}/shutdown", + health_url=f"{base_url}/health", + chat_url=f"{base_url}/v1/chat/completions", + startup_downloads=False, + auto_model_load=False, + notes=[ + "The workbench does not start SGLang on app startup.", + "Run the command explicitly after installing SGLang and choosing hardware.", + "MiniCPM tool parsing is configured through --tool-call-parser.", + ], + ) diff --git a/models/transformers_text.py b/models/transformers_text.py new file mode 100644 index 0000000000000000000000000000000000000000..4eaed91727c5f1787d9de26d2beb936a1c50b459 --- /dev/null +++ b/models/transformers_text.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass +from typing import Any, cast + +from models.base import BackendStatus +from models.hf_components import load_tokenizer_and_causal_lm +from models.model_catalog import ModelInfo + + +@dataclass(frozen=True) +class TransformersTextConfig: + trust_remote_code: bool = False + device_map: str = "auto" + torch_dtype: str = "auto" + max_new_tokens: int = 256 + temperature: float = 0.7 + do_sample: bool = True + + +class TransformersTextService: + """Optional Transformers text backend with lazy model loading.""" + + def __init__( + self, + model: ModelInfo, + config: TransformersTextConfig | None = None, + ) -> None: + self.model = model + self.config = config or TransformersTextConfig( + trust_remote_code=model.trust_remote_code + ) + self._model = None + self._tokenizer = None + + @staticmethod + def status() -> BackendStatus: + if importlib.util.find_spec("transformers") is None: + return BackendStatus( + "transformers", + False, + "Python package transformers is not installed in the current environment.", + ) + return BackendStatus("transformers", True, "Transformers package is installed.") + + def chat(self, system_prompt: str, user_prompt: str) -> str: + status = self.status() + if not status.available: + return ( + "[Transformers unavailable]\n\n" + f"{status.detail}\n\n" + "Install transformers/torch and select this backend only when local hardware " + "can load the chosen model." + ) + + model, tokenizer = self._load_components() + prompt = self._format_chat_prompt(tokenizer, system_prompt, user_prompt) + encoded = tokenizer(prompt, return_tensors="pt") + encoded = self._move_encoded_to_model_device(encoded, model) + outputs = model.generate(**encoded, **self.generation_kwargs()) + decoded = cast(str, tokenizer.decode(outputs[0], skip_special_tokens=True)) + return decoded[len(prompt) :].strip() or decoded.strip() + + def stream_chat(self, system_prompt: str, user_prompt: str) -> list[str]: + response = self.chat(system_prompt, user_prompt) + return [token for token in response.split(" ") if token] + + def generation_kwargs(self) -> dict[str, Any]: + return { + "max_new_tokens": self.config.max_new_tokens, + "temperature": self.config.temperature, + "do_sample": self.config.do_sample, + } + + def _load_components(self): + if self._model is not None and self._tokenizer is not None: + return self._model, self._tokenizer + + self._model, self._tokenizer = load_tokenizer_and_causal_lm( + self.model, + self.config.trust_remote_code, + self.config.device_map, + self.config.torch_dtype, + ) + return self._model, self._tokenizer + + @staticmethod + def _format_chat_prompt(tokenizer, system_prompt: str, user_prompt: str) -> str: + messages = [] + if system_prompt.strip(): + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + + if hasattr(tokenizer, "apply_chat_template"): + rendered = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + return str(rendered) + + parts = [f"{message['role']}: {message['content']}" for message in messages] + parts.append("assistant:") + return "\n".join(parts) + + @staticmethod + def _move_encoded_to_model_device(encoded, model): + device = getattr(model, "device", None) + if device is None: + return encoded + if hasattr(encoded, "to"): + return encoded.to(device) + return { + key: value.to(device) if hasattr(value, "to") else value + for key, value in encoded.items() + } diff --git a/models/vllm_runner.py b/models/vllm_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e815945c34fdd7a14483b3b2280eb758af5b33 --- /dev/null +++ b/models/vllm_runner.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import importlib.util +import shutil +from collections.abc import Callable +from dataclasses import asdict, dataclass +from typing import Any + +import requests + +from models.base import BackendStatus +from models.model_catalog import ModelInfo +from models.server_helpers import host_port_args, request_openai_chat_text +from tracking.trackio_client import TrackingClient + + +@dataclass(frozen=True) +class VLLMConfig: + base_url: str = "http://127.0.0.1:8000" + host: str = "127.0.0.1" + port: int = 8000 + tensor_parallel_size: int = 1 + dtype: str = "auto" + max_model_len: int = 4096 + temperature: float = 0.7 + max_tokens: int = 512 + timeout_seconds: float = 120 + + +@dataclass(frozen=True) +class VLLMRunPlan: + start_command: list[str] + stop_note: str + health_url: str + metrics_url: str + chat_url: str + startup_downloads: bool + auto_model_load: bool + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class VLLMService: + """vLLM OpenAI-compatible client and local server command planner.""" + + def __init__( + self, + model: ModelInfo, + config: VLLMConfig | None = None, + get_func: Callable[..., requests.Response] = requests.get, + post_func: Callable[..., requests.Response] = requests.post, + ) -> None: + self.model = model + self.config = config or VLLMConfig() + self.get_func = get_func + self.post_func = post_func + + @staticmethod + def status( + base_url: str = "http://127.0.0.1:8000", + which_func: Callable[[str], str | None] = shutil.which, + find_spec: Callable[[str], object | None] = importlib.util.find_spec, + get_func: Callable[..., requests.Response] = requests.get, + ) -> BackendStatus: + has_vllm = which_func("vllm") is not None or find_spec("vllm") is not None + try: + response = get_func(f"{base_url.rstrip('/')}/health", timeout=2) + except requests.RequestException as exc: + detail = f"vLLM server is not reachable: {exc}" + if not has_vllm: + detail += " Python package vllm or vllm CLI is also not installed." + return BackendStatus("vllm", False, detail) + if response.ok: + return BackendStatus("vllm", True, "vLLM server is reachable.") + return BackendStatus("vllm", False, f"vLLM responded with HTTP {response.status_code}.") + + def run_plan(self) -> VLLMRunPlan: + return build_vllm_run_plan(self.model, self.config) + + def chat(self, system_prompt: str, user_prompt: str) -> str: + status = self.status(self.config.base_url, get_func=self.get_func) + if not status.available: + return ( + "[vLLM unavailable]\n\n" + f"{status.detail}\n\n" + "Install vLLM, start the planned local server command, then retry." + ) + cfg = self.config + sampling = (cfg.temperature, cfg.max_tokens, cfg.timeout_seconds) + return request_openai_chat_text( + self.post_func, + cfg.base_url, + self.model.hf_id, + system_prompt, + user_prompt, + *sampling, + "vLLM", + ) + + +def build_vllm_run_plan(model: ModelInfo, config: VLLMConfig | None = None) -> VLLMRunPlan: + cfg = config or VLLMConfig() + base_url = cfg.base_url.rstrip("/") + return VLLMRunPlan( + start_command=[ + "vllm", + "serve", + model.hf_id, + *host_port_args(cfg.host, cfg.port), + "--tensor-parallel-size", + str(cfg.tensor_parallel_size), + "--dtype", + cfg.dtype, + "--max-model-len", + str(cfg.max_model_len), + ], + stop_note="Stop the foreground vLLM process with Ctrl+C or your process manager.", + health_url=f"{base_url}/health", + metrics_url=f"{base_url}/metrics", + chat_url=f"{base_url}/v1/chat/completions", + startup_downloads=False, + auto_model_load=False, + ) + + +def parse_vllm_metrics(metrics_text: str) -> dict[str, float]: + parsed: dict[str, float] = {} + for line in metrics_text.splitlines(): + if not line or line.startswith("#"): + continue + name, _, value = line.partition(" ") + metric_name = name.split("{", 1)[0] + try: + parsed[metric_name] = float(value.strip()) + except ValueError: + continue + return parsed + + +def fetch_vllm_metrics( + base_url: str = "http://127.0.0.1:8000", + get_func: Callable[..., requests.Response] = requests.get, +) -> dict[str, float]: + response = get_func(f"{base_url.rstrip('/')}/metrics", timeout=5) + response.raise_for_status() + return parse_vllm_metrics(response.text) + + +def log_vllm_benchmark( + metrics: dict[str, float], + model_id: str, + client: TrackingClient | None = None, +) -> str: + tracking_client = client or TrackingClient() + path = tracking_client.log( + "vllm_benchmark", + { + "model_id": model_id, + "metrics": metrics, + }, + ) + return str(path) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..39c0938d727bbe9aaed7471238dc61a14eaa381a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,78 @@ +{ + "name": "openbmb-local-ai-workbench-e2e", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openbmb-local-ai-workbench-e2e", + "version": "0.1.0", + "devDependencies": { + "@playwright/test": "^1.53.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.60.0.tgz", + "integrity": "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.60.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz", + "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.60.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000000000000000000000000000000000000..a48c078b5e9ba402cf8ce31ef6cffd56c4d7e664 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "openbmb-local-ai-workbench-e2e", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "e2e": "npm run e2e:workbench && npm run e2e:plant", + "e2e:workbench": "playwright test --config playwright.config.ts", + "e2e:plant": "powershell -ExecutionPolicy Bypass -File scripts/run_plant_e2e.ps1", + "e2e:headed": "playwright test --headed", + "e2e:record": "playwright codegen http://127.0.0.1:7860", + "e2e:install": "playwright install chromium" + }, + "devDependencies": { + "@playwright/test": "^1.53.0" + } +} diff --git a/plant/__init__.py b/plant/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d5fb3a2d5143a33714492cd8ed80eb038db81f1b --- /dev/null +++ b/plant/__init__.py @@ -0,0 +1 @@ +"""Plant Discovery reference app built from the workbench template.""" diff --git a/plant/app.py b/plant/app.py new file mode 100644 index 0000000000000000000000000000000000000000..604f1d84ae6d9ab992dd4c5785272566893a00b4 --- /dev/null +++ b/plant/app.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any, cast + +import gradio as gr +import yaml + +from core.deployment import current_policy, ensure_demo_mode_allowed +from datasets.field_notes import FieldNoteStore +from plant.plant_loader import SpeciesIndexBuilder +from plant.plant_service import DemoPlantVisionService, PlantVisionService +from plant.plant_tab import build_plant_tab +from plant.plant_tools import set_services + +ROOT = Path(__file__).parent +DEFAULT_CONFIG = ROOT / "models.yaml" + +APP_CSS = """ +.plant-shell { + max-width: 1120px !important; +} +.plant-title { + margin-bottom: 0.35rem; +} +footer { + display: none !important; +} +""" + + +def load_config(path: str | Path = DEFAULT_CONFIG) -> dict[str, Any]: + config_path = Path(path) + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + raise ValueError(f"Plant config must be a mapping: {config_path}") + return data + + +def build_app( + config_path: str | Path = DEFAULT_CONFIG, + no_model: bool = False, + data_dir: str | Path = "data", + model_mode: str = "openbmb", +) -> gr.Blocks: + policy = current_policy() + cfg = load_config(config_path) + root = Path(config_path).parent + species_index = SpeciesIndexBuilder(root=root).build(cfg) + note_store = FieldNoteStore(Path(data_dir) / "plant_field_notes.csv") + + plant_service: Any + if no_model or model_mode == "demo": + ensure_demo_mode_allowed(policy) + plant_service = DemoPlantVisionService() + elif model_mode == "finetuned": + plant_service = PlantVisionService.from_config(config_path, "plant_vlm_finetuned") + else: + plant_service = PlantVisionService.from_config(config_path, "plant_vlm") + + set_services(plant_service, note_store, species_index) + + domain = cfg.get("domain", {}) + title = str(domain.get("title") or "Plant Discovery") + description = str( + domain.get("description") + or "Identify plants, correct mistakes, and export local training data." + ) + + with gr.Blocks( + title=title, + analytics_enabled=False, + ) as demo: + gr.Markdown( + f"# {title}\n\n{description}", + elem_classes=["plant-title"], + ) + gr.Markdown( + "Local-first reference app generated around the OpenBMB Workbench template. " + "Model loading happens only after an explicit identify action." + ) + build_plant_tab(plant_service, note_store, species_index) + + return cast(gr.Blocks, demo) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Plant Discovery reference app") + parser.add_argument("--config", default=str(DEFAULT_CONFIG)) + parser.add_argument("--port", type=int, default=7861) + parser.add_argument("--share", action="store_true") + parser.add_argument( + "--model-mode", + choices=["openbmb", "finetuned", "demo"], + default="openbmb", + help=( + "openbmb uses MiniCPM-V, finetuned uses the configured adapter, " + "demo is deterministic." + ), + ) + parser.add_argument( + "--no-model", + action="store_true", + help="Use deterministic demo identification instead of loading a vision model.", + ) + args = parser.parse_args() + + model_mode = "demo" if args.no_model else args.model_mode + demo = build_app(args.config, no_model=args.no_model, model_mode=model_mode) + print(f"Starting Plant Discovery on http://127.0.0.1:{args.port}") + demo.launch( + server_port=args.port, + share=args.share, + theme=gr.themes.Soft(primary_hue="green", neutral_hue="slate"), + css=APP_CSS, + mcp_server=True, + ) + + +if __name__ == "__main__": + main() diff --git a/plant/models.yaml b/plant/models.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e08774f9b45aa998cc6eed67143e3fb1ff1e1819 --- /dev/null +++ b/plant/models.yaml @@ -0,0 +1,104 @@ +domain: + name: plant_discovery + title: "Plant Discovery" + description: "Identify plants, correct mistakes, and export local training data." + version: "0.1.0" + languages: ["en", "de", "fr", "es", "la"] + +models: + plant_vlm: + hf_id: openbmb/MiniCPM-V-4.6 + type: vision + architecture: minicpm_v + context_length: 262144 + dtype: auto + trust_remote_code: true + thinking_mode: false + role: primary + gguf: + available: true + repo: openbmb/MiniCPM-V-4.6-gguf + main_file: MiniCPM-V-4.6-Q4_K_M.gguf + mmproj: mmproj-MiniCPM-V-4.6-F16.gguf + default_quant: Q4_K_M + + plant_vlm_thinking: + hf_id: openbmb/MiniCPM-V-4.6-Thinking + type: vision + architecture: minicpm_v + context_length: 262144 + dtype: auto + trust_remote_code: true + thinking_mode: true + role: thinking_fallback + + plant_vlm_finetuned: + hf_id: your-username/minicpm-v46-plant-lora + base_model: openbmb/MiniCPM-V-4.6 + adapter_id: your-username/minicpm-v46-plant-lora + type: vision + architecture: minicpm_v + role: finetuned + requires_training: true + +inference: + default_model: plant_vlm + temperature: 0.1 + max_new_tokens: 512 + confidence_threshold: 0.70 + auto_thinking: true + multi_image_max: 4 + +datasets: + plantnet_300k: + hf_id: plantnet/plantnet_300K + split_train: train + split_val: val + split_test: test + trust_remote_code: true + image_col: image + label_col: label + n_species: 1081 + n_images: 306146 + license: CC-BY-SA-4.0 + + local_field_guide: + path: data/field_guide + format: image_folder + image_ext: [".jpg", ".jpeg", ".png", ".webp"] + +training: + lora: + rank: 16 + alpha: 32 + dropout: 0.05 + target_modules: ["q_proj", "k_proj", "v_proj", "o_proj"] + frozen_prefixes: ["vpm", "resampler", "vision_model"] + + sft: + epochs: 3 + batch_size: 4 + grad_accum: 4 + lr: 2.0e-4 + bf16: true + report_to: trackio + save_steps: 200 + eval_steps: 200 + max_seq_length: 2048 + +trackio: + project: plant-discovery + space_id: null + +taxonomy: + toxic_families: + - Solanaceae + - Euphorbiaceae + - Ranunculaceae + - Apiaceae + - Araceae + edible_families: + - Rosaceae + - Fabaceae + - Asteraceae + - Poaceae diff --git a/plant/plant_loader.py b/plant/plant_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..e76120663da6759ca8997a48a7bcae9df4c399d6 --- /dev/null +++ b/plant/plant_loader.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import csv +import json +import sqlite3 +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class PlantRecord: + image_path: str = "" + latin_name: str = "" + common_name: str = "" + family: str = "" + genus: str = "" + source: str = "" + split: str = "train" + extra: dict[str, Any] = field(default_factory=dict) + + def to_training_row(self) -> dict[str, Any]: + response = { + "common_name": self.common_name or self.latin_name, + "latin_name": self.latin_name, + "family": self.family, + "genus": self.genus or genus_from_latin(self.latin_name), + "confidence": 1.0, + "key_features": self.extra.get("key_features", []), + "care_tips": self.extra.get("care_tips", []), + "toxicity": self.extra.get("toxicity", {"humans": "unknown", "pets": "unknown"}), + "habitat": self.extra.get("habitat", ""), + "bloom_season": self.extra.get("bloom_season", ""), + "similar_species": self.extra.get("similar_species", []), + "notes": self.extra.get("notes", ""), + } + return { + "image_path": self.image_path, + "instruction": "Identify this plant species from the image.", + "response": json.dumps(response, ensure_ascii=False), + "latin_name": self.latin_name, + "family": self.family, + "source": self.source, + "split": self.split, + } + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +class LocalFolderLoader: + """Load image metadata from folders named after species.""" + + def __init__(self, root: str | Path) -> None: + self.root = Path(root) + + def iter_records( + self, + extensions: tuple[str, ...] = (".jpg", ".jpeg", ".png", ".webp"), + ) -> list[PlantRecord]: + if not self.root.exists(): + return [] + + records: list[PlantRecord] = [] + for species_dir in sorted(self.root.iterdir()): + if not species_dir.is_dir(): + continue + latin_name = species_dir.name.replace("_", " ") + for image_path in sorted(species_dir.rglob("*")): + if image_path.suffix.lower() not in extensions: + continue + records.append( + PlantRecord( + image_path=str(image_path), + latin_name=latin_name, + common_name=latin_name, + genus=genus_from_latin(latin_name), + source="local_folder", + ) + ) + return records + + def species_list(self) -> list[str]: + if not self.root.exists(): + return [] + return [ + path.name.replace("_", " ") + for path in sorted(self.root.iterdir()) + if path.is_dir() + ] + + +class GBIFLoader: + """Load a small Darwin Core TSV export for metadata enrichment.""" + + def __init__(self, csv_path: str | Path) -> None: + self.csv_path = Path(csv_path) + + def load_metadata(self) -> dict[str, dict[str, Any]]: + if not self.csv_path.exists(): + return {} + + metadata: dict[str, dict[str, Any]] = {} + with self.csv_path.open(newline="", encoding="utf-8") as handle: + reader = csv.DictReader(handle, delimiter="\t") + for row in reader: + species = row.get("species", "").strip() + if not species: + continue + entry = metadata.setdefault( + species, + { + "family": row.get("family", ""), + "genus": row.get("genus", ""), + "countries": set(), + }, + ) + country = row.get("country", "").strip() + if country: + entry["countries"].add(country) + + for value in metadata.values(): + value["countries"] = sorted(value["countries"]) + return metadata + + +class SpeciesIndexBuilder: + """Build an in-memory plant species index without network by default.""" + + def __init__(self, root: str | Path = "plant") -> None: + self.root = Path(root) + + def build(self, config: dict[str, Any]) -> dict[str, dict[str, Any]]: + index = self._from_local_folder(config) + self._enrich_from_cached_labels(index) + self._enrich_from_gbif(index) + return index or demo_species() + + def _from_local_folder(self, config: dict[str, Any]) -> dict[str, dict[str, Any]]: + dataset_cfg = config.get("datasets", {}).get("local_field_guide", {}) + configured_path = Path(str(dataset_cfg.get("path", "data/field_guide"))) + local_path = ( + configured_path + if configured_path.is_absolute() + else self.root / configured_path + ) + loader = LocalFolderLoader(local_path) + index: dict[str, dict[str, Any]] = {} + for species in loader.species_list(): + index[species] = { + "common_name": species, + "family": "", + "genus": genus_from_latin(species), + "n_images": 0, + "source": "local_folder", + } + return index + + def _enrich_from_cached_labels(self, index: dict[str, dict[str, Any]]) -> None: + labels_path = self.root / "data" / "plantnet_labels.json" + if not labels_path.exists(): + return + labels = json.loads(labels_path.read_text(encoding="utf-8")) + if not isinstance(labels, dict): + return + for latin_name, meta in labels.items(): + if not isinstance(meta, dict): + continue + entry = index.setdefault( + latin_name, + { + "common_name": latin_name, + "family": "", + "genus": genus_from_latin(latin_name), + "n_images": 0, + "source": "plantnet_cache", + }, + ) + for key, value in meta.items(): + if value and not entry.get(key): + entry[key] = value + + def _enrich_from_gbif(self, index: dict[str, dict[str, Any]]) -> None: + metadata = GBIFLoader(self.root / "data" / "gbif_occurrences.tsv").load_metadata() + for latin_name, values in metadata.items(): + if latin_name not in index: + continue + index[latin_name].setdefault("habitat_countries", values.get("countries", [])) + if values.get("family") and not index[latin_name].get("family"): + index[latin_name]["family"] = values["family"] + + +class FieldNotesPlantExporter: + """Read corrected field notes and export plant training rows.""" + + def __init__(self, csv_path: str | Path = "data/plant_field_notes.csv") -> None: + self.csv_path = Path(csv_path) + + def load_corrections(self) -> list[PlantRecord]: + if not self.csv_path.exists(): + return [] + + with self.csv_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + + records: list[PlantRecord] = [] + for row in rows: + correction = row.get("correction", "").strip() + if not correction: + continue + records.append( + PlantRecord( + image_path=row.get("image_path", ""), + latin_name=correction, + common_name=correction, + genus=genus_from_latin(correction), + source="field_notes", + extra={"original_prediction": row.get("response", "")}, + ) + ) + return records + + def export_jsonl(self, output_path: str | Path = "data/plant_training.jsonl") -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8") as handle: + for record in self.load_corrections(): + handle.write(json.dumps(record.to_training_row(), ensure_ascii=False) + "\n") + return output + + +class SQLitePlantNoteReader: + """Compatibility reader for old plant note SQLite files.""" + + def __init__(self, db_path: str | Path = "data/field_notes.db") -> None: + self.db_path = Path(db_path) + + def count_rows(self) -> int: + if not self.db_path.exists(): + return 0 + connection = sqlite3.connect(self.db_path) + try: + row = connection.execute("SELECT COUNT(*) FROM notes").fetchone() + return int(row[0]) if row else 0 + finally: + connection.close() + + +def demo_species() -> dict[str, dict[str, Any]]: + return { + "Acer palmatum": { + "common_name": "Japanese Maple", + "family": "Sapindaceae", + "genus": "Acer", + "source": "demo", + }, + "Bellis perennis": { + "common_name": "Common Daisy", + "family": "Asteraceae", + "genus": "Bellis", + "source": "demo", + }, + "Rosa canina": { + "common_name": "Dog Rose", + "family": "Rosaceae", + "genus": "Rosa", + "source": "demo", + }, + "Quercus robur": { + "common_name": "English Oak", + "family": "Fagaceae", + "genus": "Quercus", + "source": "demo", + }, + "Urtica dioica": { + "common_name": "Stinging Nettle", + "family": "Urticaceae", + "genus": "Urtica", + "source": "demo", + }, + } + + +def genus_from_latin(latin_name: str) -> str: + return latin_name.split()[0] if " " in latin_name else latin_name diff --git a/plant/plant_service.py b/plant/plant_service.py new file mode 100644 index 0000000000000000000000000000000000000000..0d0bacdefb8cc64fb095e621298ab2dd0160f5ad --- /dev/null +++ b/plant/plant_service.py @@ -0,0 +1,429 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import re +import time +from dataclasses import asdict, dataclass, field +from importlib import import_module +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass +class PlantID: + common_name: str = "Unknown" + latin_name: str = "Unknown" + family: str = "Unknown" + genus: str = "Unknown" + confidence: float = 0.0 + key_features: list[str] = field(default_factory=list) + care_tips: list[str] = field(default_factory=list) + toxicity: dict[str, str] = field( + default_factory=lambda: {"humans": "unknown", "pets": "unknown"} + ) + habitat: str = "" + bloom_season: str = "" + similar_species: list[str] = field(default_factory=list) + notes: str = "" + thinking_trace: str = "" + latency_ms: float = 0.0 + model_used: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def is_uncertain(self, threshold: float = 0.70) -> bool: + return self.confidence < threshold + + @classmethod + def from_mapping(cls, data: dict[str, Any], model_used: str = "") -> PlantID: + latin_name = str(data.get("latin_name") or "Unknown") + genus = str(data.get("genus") or _genus_from_latin(latin_name)) + return cls( + common_name=str(data.get("common_name") or latin_name), + latin_name=latin_name, + family=str(data.get("family") or "Unknown"), + genus=genus, + confidence=_clamp_float(data.get("confidence"), 0.0, 1.0), + key_features=_string_list(data.get("key_features")), + care_tips=_string_list(data.get("care_tips")), + toxicity=_toxicity_mapping(data.get("toxicity")), + habitat=str(data.get("habitat") or ""), + bloom_season=str(data.get("bloom_season") or ""), + similar_species=_string_list(data.get("similar_species")), + notes=str(data.get("notes") or ""), + model_used=model_used, + ) + + +@dataclass(frozen=True) +class PlantVisionConfig: + model_id: str = "openbmb/MiniCPM-V-4.6" + thinking_model_id: str = "openbmb/MiniCPM-V-4.6-Thinking" + adapter_id: str = "" + model_key: str = "plant_vlm" + confidence_threshold: float = 0.70 + max_new_tokens: int = 512 + temperature: float = 0.1 + trust_remote_code: bool = True + device_map: str = "auto" + torch_dtype: str = "auto" + auto_thinking: bool = True + + +class DemoPlantVisionService: + """Deterministic no-model service for screenshots, tests, and template demos.""" + + model_id = "demo/plant-discovery" + + def service_status(self) -> dict[str, Any]: + return { + "mode": "demo", + "uses_llm": False, + "model_id": self.model_id, + "ready": True, + "notes": "Deterministic demo mode; no model weights are loaded.", + } + + def identify( + self, + image: Any, + extra_images: list[Any] | None = None, + force_thinking: bool = False, + ) -> PlantID: + image_count = 1 + len(extra_images or []) + confidence = 0.86 if force_thinking or image_count > 1 else 0.72 + return PlantID( + common_name="Common Daisy", + latin_name="Bellis perennis", + family="Asteraceae", + genus="Bellis", + confidence=confidence, + key_features=[ + "white ray florets around yellow disc", + "low basal rosette", + "small lawn-forming habit", + ], + care_tips=[ + "Prefers full sun to partial shade", + "Tolerates compact lawn soil", + "Water during long dry periods", + ], + toxicity={"humans": "non-toxic", "pets": "non-toxic"}, + habitat="Lawns, meadows, paths, and disturbed grassland", + bloom_season="spring to autumn", + similar_species=["Leucanthemum vulgare", "Erigeron annuus"], + notes="Demo result for the template app. Replace with a real MiniCPM-V backend.", + thinking_trace="Demo mode used a deterministic plant card." if force_thinking else "", + latency_ms=25.0, + model_used=self.model_id, + ) + + +class PlantVisionService: + """Optional MiniCPM-V plant identification adapter. + + The class is importable without torch/transformers. Heavy dependencies and model weights are + touched only when identify() is called. + """ + + def __init__(self, config: PlantVisionConfig | None = None) -> None: + self.config = config or PlantVisionConfig() + self.processor: Any | None = None + self.model: Any | None = None + self.thinking_processor: Any | None = None + self.thinking_model: Any | None = None + + @staticmethod + def dependency_report() -> dict[str, bool]: + return { + "transformers": importlib.util.find_spec("transformers") is not None, + "torch": importlib.util.find_spec("torch") is not None, + "pillow": importlib.util.find_spec("PIL") is not None, + "peft": importlib.util.find_spec("peft") is not None, + } + + @classmethod + def from_config( + cls, + config_path: str | Path, + model_key: str = "plant_vlm", + ) -> PlantVisionService: + cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) or {} + models = cfg.get("models", {}) + inference = cfg.get("inference", {}) + primary = models.get(model_key, {}) or models.get("plant_vlm", {}) + thinking = models.get("plant_vlm_thinking", {}) + adapter_id = str(primary.get("adapter_id") or "") + if not adapter_id and bool(primary.get("requires_training", False)): + adapter_id = str(primary.get("hf_id") or "") + return cls( + PlantVisionConfig( + model_id=str( + primary.get("base_model") + or primary.get("hf_id") + or PlantVisionConfig.model_id + ), + thinking_model_id=str( + thinking.get("hf_id") or PlantVisionConfig.thinking_model_id + ), + adapter_id=adapter_id, + model_key=model_key, + confidence_threshold=float(inference.get("confidence_threshold", 0.70)), + max_new_tokens=int( + os.getenv("PLANT_MAX_NEW_TOKENS", str(inference.get("max_new_tokens", 512))) + ), + temperature=float(inference.get("temperature", 0.1)), + trust_remote_code=bool(primary.get("trust_remote_code", True)), + torch_dtype=str(primary.get("dtype") or "auto"), + auto_thinking=_env_bool( + "PLANT_AUTO_THINKING", + bool(inference.get("auto_thinking", True)), + ), + ) + ) + + def service_status(self) -> dict[str, Any]: + dependencies = self.dependency_report() + required = ["transformers", "torch", "pillow"] + if self.config.adapter_id: + required.append("peft") + missing = [name for name in required if not dependencies[name]] + return { + "mode": "openbmb-finetuned" if self.config.adapter_id else "openbmb-zero-shot", + "uses_llm": True, + "model_id": self.config.model_id, + "adapter_id": self.config.adapter_id, + "model_key": self.config.model_key, + "ready": not missing, + "missing_dependencies": missing, + "notes": ( + "Uses OpenBMB MiniCPM-V through Transformers. Weights load only during Identify." + ), + } + + def identify( + self, + image: Any, + extra_images: list[Any] | None = None, + force_thinking: bool = False, + ) -> PlantID: + status = self.service_status() + missing = list(status["missing_dependencies"]) + if missing: + return PlantID( + common_name="Model unavailable", + latin_name="MiniCPM-V dependencies missing", + confidence=0.0, + notes=( + "Install optional plant model dependencies before real inference: " + + ", ".join(missing) + ), + model_used=self.config.model_id, + ) + + start = time.perf_counter() + try: + result = self._identify_with_model(image, extra_images, force_thinking) + except (ImportError, RuntimeError, OSError, ValueError) as exc: + return PlantID( + common_name="Model unavailable", + latin_name="MiniCPM-V could not run", + confidence=0.0, + notes=str(exc), + model_used=self.config.model_id, + ) + result.latency_ms = (time.perf_counter() - start) * 1000 + return result + + def _identify_with_model( + self, + image: Any, + extra_images: list[Any] | None, + force_thinking: bool, + ) -> PlantID: + images = [image, *(extra_images or [])][:4] + if force_thinking: + return self._infer(images, thinking=True) + + result = self._infer(images, thinking=False) + if self.config.auto_thinking and result.is_uncertain(self.config.confidence_threshold): + thinking_result = self._infer(images, thinking=True) + if thinking_result.confidence > result.confidence: + return thinking_result + return result + + def _infer(self, images: list[Any], thinking: bool) -> PlantID: + torch = import_module("torch") + model, processor = self._load_components(thinking=thinking) + messages = build_plant_messages(images, thinking=thinking) + inputs = processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_tensors="pt", + return_dict=True, + chat_template_kwargs={"enable_thinking": thinking}, + ) + device = getattr(model, "device", None) + if device is not None: + inputs = {key: value.to(device) for key, value in inputs.items()} + + with torch.no_grad(): + output_ids = model.generate( + **inputs, + max_new_tokens=self.config.max_new_tokens if not thinking else 1024, + temperature=self.config.temperature if not thinking else 0.2, + do_sample=True, + ) + + raw = processor.decode( + output_ids[0][inputs["input_ids"].shape[1] :], + skip_special_tokens=True, + ) + model_id = self.config.thinking_model_id if thinking else self.config.model_id + parsed = parse_plant_response(raw, model_used=model_id) + if thinking: + parsed.thinking_trace = extract_thinking_trace(raw) + return parsed + + def _load_components(self, thinking: bool) -> tuple[Any, Any]: + transformers = import_module("transformers") + model_id = self.config.thinking_model_id if thinking else self.config.model_id + + if thinking and self.thinking_model is not None and self.thinking_processor is not None: + return self.thinking_model, self.thinking_processor + if not thinking and self.model is not None and self.processor is not None: + return self.model, self.processor + + processor = transformers.AutoProcessor.from_pretrained( + model_id, + trust_remote_code=self.config.trust_remote_code, + ) + model = transformers.AutoModelForImageTextToText.from_pretrained( + model_id, + trust_remote_code=self.config.trust_remote_code, + device_map=self.config.device_map, + torch_dtype=self.config.torch_dtype, + ) + if self.config.adapter_id and not thinking: + peft = import_module("peft") + model = peft.PeftModel.from_pretrained(model, self.config.adapter_id) + model.eval() + + if thinking: + self.thinking_model = model + self.thinking_processor = processor + else: + self.model = model + self.processor = processor + return model, processor + + +def build_plant_messages(images: list[Any], thinking: bool = False) -> list[dict[str, Any]]: + prompt = ( + "This is a difficult plant identification. Compare leaf shape, flowers, fruit, stem, " + "and growth habit before answering." + if thinking + else "Identify this plant species from the image." + ) + if len(images) > 1: + prompt = f"Identify the plant using these {len(images)} images from different angles." + + content = [{"type": "image", "image": image} for image in images[:4]] + content.append({"type": "text", "text": prompt + " Respond only as valid JSON."}) + return [ + {"role": "system", "content": plant_system_prompt()}, + {"role": "user", "content": content}, + ] + + +def plant_system_prompt() -> str: + return ( + "You are a careful botanist. Return only JSON with keys: common_name, latin_name, " + "family, genus, confidence, key_features, care_tips, toxicity, habitat, " + "bloom_season, similar_species, notes." + ) + + +def parse_plant_response(raw: str, model_used: str = "") -> PlantID: + data = extract_json_object(raw) + if "error" in data: + return PlantID( + notes=f"Could not parse model JSON: {data.get('raw', '')}", + model_used=model_used, + ) + return PlantID.from_mapping(data, model_used=model_used) + + +def extract_json_object(text: str) -> dict[str, Any]: + stripped = text.strip() + for candidate in _json_candidates(stripped): + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + repaired = re.sub(r",\s*([}\]])", r"\1", candidate) + try: + parsed = json.loads(repaired) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + return parsed + return {"error": "unparseable", "raw": stripped[:300]} + + +def extract_thinking_trace(text: str) -> str: + match = re.search(r"(.*?)", text, re.DOTALL) + return match.group(1).strip() if match else "" + + +def _json_candidates(text: str) -> list[str]: + candidates = [text] + fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL) + if fence: + candidates.append(fence.group(1)) + start = text.find("{") + end = text.rfind("}") + if start >= 0 and end > start: + candidates.append(text[start : end + 1]) + return candidates + + +def _genus_from_latin(latin_name: str) -> str: + return latin_name.split()[0] if " " in latin_name else latin_name + + +def _string_list(value: Any) -> list[str]: + if isinstance(value, list): + return [str(item) for item in value if str(item).strip()] + if isinstance(value, str) and value.strip(): + return [value.strip()] + return [] + + +def _toxicity_mapping(value: Any) -> dict[str, str]: + if isinstance(value, dict): + return { + "humans": str(value.get("humans") or "unknown"), + "pets": str(value.get("pets") or "unknown"), + } + return {"humans": "unknown", "pets": "unknown"} + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _clamp_float(value: Any, minimum: float, maximum: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return minimum + return min(max(parsed, minimum), maximum) diff --git a/plant/plant_tab.py b/plant/plant_tab.py new file mode 100644 index 0000000000000000000000000000000000000000..9eae1074ba6ae75fe5bd11e897bfc95ebc8aace0 --- /dev/null +++ b/plant/plant_tab.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import html +from pathlib import Path +from typing import Any + +import gradio as gr + +from core.spaces_runtime import spaces +from datasets.field_notes import FieldNote, FieldNoteStore +from plant.plant_loader import FieldNotesPlantExporter +from plant.plant_service import PlantID +from plant.training import build_plant_training_plan + + +def build_plant_tab( + plant_service: Any, + note_store: FieldNoteStore, + species_index: dict[str, dict[str, Any]], +) -> None: + with gr.Tabs(): + with gr.Tab("Identify"): + build_identify_panel(plant_service, note_store) + with gr.Tab("Field Guide"): + build_field_guide_panel(species_index) + with gr.Tab("Corrections"): + build_corrections_panel(note_store) + with gr.Tab("Stats"): + build_stats_panel(note_store, species_index) + + +def build_identify_panel(plant_service: Any, note_store: FieldNoteStore) -> None: + status = ( + plant_service.service_status() + if hasattr(plant_service, "service_status") + else {"mode": "unknown", "uses_llm": False} + ) + gr.JSON(status, label="Model status") + with gr.Row(): + with gr.Column(scale=1): + image = gr.Image(type="pil", label="Primary plant image", height=280) + extra_images = gr.Gallery( + label="Extra angles", + type="pil", + columns=2, + height=180, + ) + mode = gr.Dropdown( + ["standard", "thinking"], + value="standard", + label="Analysis mode", + ) + identify = gr.Button("Identify plant", variant="primary") + with gr.Column(scale=1): + result_card = gr.HTML(label="Result") + result_json = gr.JSON(label="Structured result") + correction = gr.Textbox(label="Human correction", placeholder="Genus species") + notes = gr.Textbox(label="Correction notes", lines=2) + save = gr.Button("Save correction / field note") + save_status = gr.Markdown() + + last_result = gr.State({}) + + identify.click( + lambda img, gallery, selected_mode: identify_plant_callback( + plant_service, + img, + gallery, + selected_mode, + ), + [image, extra_images, mode], + [result_card, result_json, last_result], + ) + save.click( + lambda result, corrected, note_text: save_field_note_callback( + note_store, + result, + corrected, + note_text, + ), + [last_result, correction, notes], + save_status, + ) + + +def build_field_guide_panel(species_index: dict[str, dict[str, Any]]) -> None: + query = gr.Textbox(label="Search", placeholder="Common name, Latin name, family") + family = gr.Dropdown( + choices=available_families(species_index), + label="Family", + allow_custom_value=True, + ) + search = gr.Button("Search species", variant="primary") + table = gr.Dataframe( + headers=["Latin name", "Common name", "Family", "Genus", "Source"], + interactive=False, + wrap=True, + ) + detail = gr.HTML(label="Species detail") + + search.click(lambda text, fam: species_table(species_index, text, fam), [query, family], table) + + def on_species_select(rows: list[list[str]], event: gr.SelectData) -> str: + return species_detail(species_index, event, rows) + + table.select(on_species_select, table, detail) + + +def build_corrections_panel(note_store: FieldNoteStore) -> None: + gr.Markdown("Corrections are saved locally and can be exported for later LoRA training.") + refresh = gr.Button("Load corrections") + export = gr.Button("Export training JSONL", variant="primary") + training = gr.Button("Prepare training plan") + rows = gr.Dataframe( + headers=["created_at", "prediction", "correction", "tags", "image_path"], + interactive=False, + wrap=True, + ) + status = gr.JSON(label="Export or training plan") + + refresh.click(lambda: correction_rows(note_store), outputs=rows) + export.click(lambda: export_corrections(note_store), outputs=status) + training.click(lambda: plant_training_plan(note_store), outputs=status) + + +def build_stats_panel( + note_store: FieldNoteStore, + species_index: dict[str, dict[str, Any]], +) -> None: + refresh = gr.Button("Refresh stats") + stats = gr.JSON(label="Plant app stats") + refresh.click(lambda: plant_stats(note_store, species_index), outputs=stats) + + +@spaces.GPU(duration=240) +def identify_plant_callback( + plant_service: Any, + image: Any, + gallery: list[Any] | None, + mode: str, +) -> tuple[str, dict[str, Any], dict[str, Any]]: + if image is None: + empty = { + "common_name": "No image", + "latin_name": "Upload a plant image", + "confidence": 0.0, + "notes": "Add a photo before identifying.", + } + return render_result_card(empty), empty, empty + + extras = normalize_gallery_images(gallery) + result = plant_service.identify( + image, + extra_images=extras, + force_thinking=(mode == "thinking"), + ) + result_data = result.to_dict() if isinstance(result, PlantID) else dict(result) + return render_result_card(result_data), result_data, result_data + + +def save_field_note_callback( + note_store: FieldNoteStore, + result: dict[str, Any], + correction: str, + notes: str, +) -> str: + if not result: + return "Run an identification before saving a field note." + predicted = str(result.get("latin_name") or "Unknown") + note = FieldNote.create( + model_id=str(result.get("model_used") or "plant_vlm"), + prompt="Identify this plant species from the image.", + response=predicted, + correction=correction.strip(), + tags="plant-discovery,correction" if correction.strip() else "plant-discovery", + image_path=str(result.get("image_path") or ""), + use_for_training=bool(correction.strip()), + ) + note_store.save(note) + suffix = f" Correction: {correction.strip()}." if correction.strip() else "" + note_hint = f" Notes: {notes.strip()}." if notes.strip() else "" + return f"Saved field note for {predicted}.{suffix}{note_hint}" + + +def render_result_card(result: dict[str, Any]) -> str: + confidence = _confidence(result.get("confidence")) + confidence_percent = int(confidence * 100) + color = "#0f766e" if confidence >= 0.80 else "#a16207" if confidence >= 0.55 else "#b91c1c" + features = "".join( + f"
  • {html.escape(str(feature))}
    • " + for feature in result.get("key_features", [])[:5] + ) + care = "".join( + f"
  • {html.escape(str(tip))}
    • " + for tip in result.get("care_tips", [])[:4] + ) + common_name = html.escape(str(result.get("common_name", "Unknown"))) + latin_name = html.escape(str(result.get("latin_name", ""))) + family = html.escape(str(result.get("family", ""))) + genus = html.escape(str(result.get("genus", ""))) + notes = html.escape(str(result.get("notes", ""))) + return f""" +
    +
    +
    +

    {common_name}

    +

    + {latin_name} +

    +

    + {family} / {genus} +

    +
    +
    + {confidence_percent}% +
    +
    +
    +
    +
    +
    +
    Key features
      {features}
    +
    Care tips
      {care}
    +
    +

    {notes}

    +
    +""" + + +def normalize_gallery_images(gallery: list[Any] | None) -> list[Any]: + images: list[Any] = [] + for item in gallery or []: + if isinstance(item, list | tuple) and item: + images.append(item[0]) + elif item is not None: + images.append(item) + return images[:3] + + +def available_families(species_index: dict[str, dict[str, Any]]) -> list[str]: + return sorted( + {str(meta.get("family")) for meta in species_index.values() if meta.get("family")} + ) + + +def species_table( + species_index: dict[str, dict[str, Any]], + query: str, + family: str | None, + limit: int = 100, +) -> list[list[str]]: + query_value = query.lower().strip() + family_value = (family or "").lower().strip() + rows: list[list[str]] = [] + for latin_name, meta in sorted(species_index.items()): + common = str(meta.get("common_name") or "") + family_name = str(meta.get("family") or "") + haystack = " ".join([latin_name, common, family_name]).lower() + if query_value and query_value not in haystack: + continue + if family_value and family_value != family_name.lower(): + continue + rows.append( + [ + latin_name, + common, + family_name, + str(meta.get("genus") or ""), + str(meta.get("source") or ""), + ] + ) + if len(rows) >= limit: + break + return rows + + +def species_detail( + species_index: dict[str, dict[str, Any]], + event: gr.SelectData, + rows: list[list[str]] | None, +) -> str: + if rows is None or event.index is None: + return "" + row_index = event.index[0] if isinstance(event.index, tuple) else int(event.index) + if row_index >= len(rows): + return "" + latin_name = rows[row_index][0] + meta = species_index.get(latin_name, {}) + countries = ", ".join(meta.get("habitat_countries", [])[:8]) or "unknown" + return ( + "
    " + f"{html.escape(str(meta.get('common_name') or latin_name))}" + f"

    {html.escape(latin_name)}

    " + f"

    Family: {html.escape(str(meta.get('family') or 'unknown'))}

    " + f"

    Known regions: {html.escape(countries)}

    " + "
    " + ) + + +def correction_rows(note_store: FieldNoteStore) -> list[list[str]]: + return [ + [ + note.created_at, + note.response, + note.correction or "-", + note.tags, + note.image_path, + ] + for note in note_store.list_notes() + ] + + +def export_corrections(note_store: FieldNoteStore) -> dict[str, Any]: + exporter = FieldNotesPlantExporter(note_store.path) + output = exporter.export_jsonl(Path("data") / "plant_training.jsonl") + rows = exporter.load_corrections() + return { + "output_path": str(output), + "rows": len(rows), + "execute_training": False, + "message": "Exported corrected plant notes. Training remains a separate explicit step.", + } + + +def plant_training_plan(note_store: FieldNoteStore) -> dict[str, Any]: + corrected = len(note_store.list_notes(corrected_only=True, training_only=True)) + return build_plant_training_plan(corrected_examples=corrected).to_dict() + + +def plant_stats( + note_store: FieldNoteStore, + species_index: dict[str, dict[str, Any]], +) -> dict[str, Any]: + notes = note_store.list_notes() + corrections = [note for note in notes if note.correction.strip()] + return { + "species_index_size": len(species_index), + "field_notes": len(notes), + "corrections": len(corrections), + "training_ready": len([note for note in corrections if note.use_for_training]), + } + + +def _confidence(value: Any) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return 0.0 + return min(max(parsed, 0.0), 1.0) diff --git a/plant/plant_tools.py b/plant/plant_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..9a4114b2023036d1c3310427f510b6b3c233fb37 --- /dev/null +++ b/plant/plant_tools.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from datasets.field_notes import FieldNote, FieldNoteStore +from plant.plant_loader import FieldNotesPlantExporter +from plant.training import build_plant_training_plan + +_plant_service: Any | None = None +_note_store: FieldNoteStore | None = None +_species_index: dict[str, dict[str, Any]] = {} + + +def set_services( + plant_service: Any | None, + note_store: FieldNoteStore | None, + species_index: dict[str, dict[str, Any]] | None, +) -> None: + global _plant_service, _note_store, _species_index + _plant_service = plant_service + _note_store = note_store + _species_index = species_index or {} + + +def identify_plant(image_path: str, detail_level: str = "standard") -> dict[str, Any]: + if _plant_service is None: + return {"status": "error", "message": "Plant service is not initialised."} + path = Path(image_path) + if not path.exists(): + return {"status": "error", "message": f"Image not found: {image_path}"} + result = _plant_service.identify(path, force_thinking=(detail_level == "thinking")) + return dict(result.to_dict()) + + +def search_species( + query: str, + family_filter: str | None = None, + limit: int = 10, +) -> list[dict[str, Any]]: + query_value = query.lower().strip() + family_value = (family_filter or "").lower().strip() + results: list[dict[str, Any]] = [] + for latin_name, meta in sorted(_species_index.items()): + common = str(meta.get("common_name") or "") + family = str(meta.get("family") or "") + haystack = " ".join([latin_name, common, family]).lower() + if query_value and query_value not in haystack: + continue + if family_value and family_value != family.lower(): + continue + results.append( + { + "latin_name": latin_name, + "common_name": common, + "family": family, + "genus": meta.get("genus", ""), + "source": meta.get("source", ""), + } + ) + if len(results) >= limit: + break + return results + + +def save_field_correction( + predicted_latin: str, + correct_latin: str, + image_path: str = "", + notes: str = "", +) -> dict[str, Any]: + if _note_store is None: + return {"status": "error", "message": "Field note store is not initialised."} + if not correct_latin.strip(): + return {"status": "error", "message": "A correction is required."} + + note = FieldNote.create( + model_id="plant_vlm", + prompt="Identify this plant species from the image.", + response=predicted_latin, + correction=correct_latin.strip(), + tags="plant-discovery,mcp-correction", + image_path=image_path, + use_for_training=True, + ) + _note_store.save(note) + return { + "status": "saved", + "message": f"Correction saved: {predicted_latin} -> {correct_latin}", + "notes": notes, + } + + +def export_corrections(output_path: str = "data/plant_training.jsonl") -> dict[str, Any]: + if _note_store is None: + return {"status": "error", "message": "Field note store is not initialised."} + exporter = FieldNotesPlantExporter(_note_store.path) + path = exporter.export_jsonl(output_path) + rows = exporter.load_corrections() + return { + "status": "exported", + "output_path": str(path), + "rows": len(rows), + "execute_training": False, + } + + +def dataset_stats() -> dict[str, Any]: + notes = _note_store.list_notes() if _note_store else [] + corrections = [note for note in notes if note.correction.strip()] + return { + "species_index_size": len(_species_index), + "field_notes": len(notes), + "corrections": len(corrections), + } + + +def training_plan( + dataset_path: str = "data/plant_training.jsonl", + lora_rank: int = 16, + epochs: int = 3, +) -> dict[str, Any]: + plan = build_plant_training_plan( + dataset_path=dataset_path, + corrected_examples=dataset_stats().get("corrections", 0), + ).to_dict() + plan["requested_lora_rank"] = lora_rank + plan["requested_epochs"] = epochs + return plan + + +def build_mcp_server(): + """Build a FastMCP server only when the optional mcp package is installed.""" + try: + from mcp.server.fastmcp import FastMCP + except ImportError as exc: + raise RuntimeError("Install mcp[cli] to expose Plant Discovery tools over MCP.") from exc + + mcp = FastMCP( + name="PlantDiscovery", + description="Plant identification and correction-loop tools.", + ) + mcp.tool()(identify_plant) + mcp.tool()(search_species) + mcp.tool()(save_field_correction) + mcp.tool()(export_corrections) + mcp.tool()(dataset_stats) + mcp.tool()(training_plan) + return mcp + + +if __name__ == "__main__": + server = build_mcp_server() + server.run(transport="sse", port=8081) diff --git a/plant/requirements.txt b/plant/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fa3860388e83d6006f5bd8e40271c1f1c4b03fe5 --- /dev/null +++ b/plant/requirements.txt @@ -0,0 +1,27 @@ +# Plant Discovery optional runtime. +# +# The root project can import and run the no-model Plant Discovery demo without these +# heavyweight packages. Install this file only when you want real MiniCPM-V inference +# or local vision LoRA experiments. + +gradio[mcp]>=5.38.0 +pyyaml>=6.0.0 +Pillow>=10.0.0 +huggingface_hub>=0.23.0 +requests>=2.31.0 +pandas>=2.0.0 + +# Real vision inference. +transformers>=4.41.0 +torch>=2.1.0 +torchvision +accelerate + +# Optional dataset/training path. +datasets>=2.18.0 +peft>=0.10.0 +trl>=0.9.0 + +# Optional MCP and tracking extras. +mcp[cli]>=1.0.0 +trackio>=0.1.0 diff --git a/plant/training.py b/plant/training.py new file mode 100644 index 0000000000000000000000000000000000000000..ddb9ac71980df66c908291f5855ef41d479adaa1 --- /dev/null +++ b/plant/training.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import importlib.util +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass(frozen=True) +class PlantTrainingPlan: + execute_training: bool + backend: str + base_model: str + dataset_path: str + output_dir: str + adapter_repo: str + corrected_examples: int + minimum_recommended_examples: int + dependency_report: dict[str, bool] + swift_command: list[str] + llamafactory_command: list[str] + publish_commands: list[list[str]] + use_trained_model_command: list[str] + notes: list[str] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def build_plant_training_plan( + config_path: str | Path = "plant/models.yaml", + dataset_path: str | Path = "data/plant_training.jsonl", + output_dir: str | Path = "checkpoints/plant_lora", + adapter_repo: str = "your-username/minicpm-v46-plant-lora", + corrected_examples: int = 0, +) -> PlantTrainingPlan: + cfg = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) or {} + models = cfg.get("models", {}) + training = cfg.get("training", {}) + lora = training.get("lora", {}) + sft = training.get("sft", {}) + base_model = str( + models.get("plant_vlm", {}).get("hf_id") + or models.get("plant_vlm_finetuned", {}).get("base_model") + or "openbmb/MiniCPM-V-4.6" + ) + rank = int(lora.get("rank", 16)) + epochs = int(sft.get("epochs", 3)) + batch_size = int(sft.get("batch_size", 4)) + grad_accum = int(sft.get("grad_accum", 4)) + lr = str(sft.get("lr", "2.0e-4")) + output = str(output_dir) + dataset = str(dataset_path) + + return PlantTrainingPlan( + execute_training=False, + backend="SWIFT vision LoRA preferred; LLaMA-Factory plan documented as alternative", + base_model=base_model, + dataset_path=dataset, + output_dir=output, + adapter_repo=adapter_repo, + corrected_examples=corrected_examples, + minimum_recommended_examples=30, + dependency_report=plant_training_dependency_report(), + swift_command=[ + "swift", + "sft", + "--model", + base_model, + "--dataset", + dataset, + "--lora_rank", + str(rank), + "--num_train_epochs", + str(epochs), + "--per_device_train_batch_size", + str(batch_size), + "--gradient_accumulation_steps", + str(grad_accum), + "--learning_rate", + lr, + "--freeze_vit", + "true", + "--output_dir", + output, + ], + llamafactory_command=[ + "llamafactory-cli", + "train", + "plant_lora.yaml", + ], + publish_commands=[ + [ + "huggingface-cli", + "repo", + "create", + adapter_repo, + "--type", + "model", + ], + [ + "huggingface-cli", + "upload", + adapter_repo, + output, + ".", + ], + ], + use_trained_model_command=[ + "python", + "-m", + "plant.app", + "--model-mode", + "finetuned", + "--port", + "7861", + ], + notes=[ + "This plan does not start training automatically.", + "Use OpenBMB MiniCPM-V zero-shot first, then collect corrected plant examples.", + "Train only after reviewing dataset quality and GPU memory.", + "Set models.plant_vlm_finetuned.adapter_id to your real adapter repo after upload.", + ], + ) + + +def plant_training_dependency_report() -> dict[str, bool]: + return { + "torch": importlib.util.find_spec("torch") is not None, + "transformers": importlib.util.find_spec("transformers") is not None, + "datasets": importlib.util.find_spec("datasets") is not None, + "peft": importlib.util.find_spec("peft") is not None, + "trl": importlib.util.find_spec("trl") is not None, + "swift": importlib.util.find_spec("swift") is not None, + } + + +def write_llamafactory_dataset_info( + dataset_path: str | Path = "data/plant_training.jsonl", + output_path: str | Path = "data/llamafactory_dataset_info.json", +) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + ( + "{\n" + ' "plant_discovery": {\n' + f' "file_name": "{Path(dataset_path).as_posix()}",\n' + ' "formatting": "alpaca",\n' + ' "columns": {\n' + ' "prompt": "instruction",\n' + ' "response": "response"\n' + " }\n" + " }\n" + "}\n" + ), + encoding="utf-8", + ) + return output diff --git a/plant_space_app.py b/plant_space_app.py new file mode 100644 index 0000000000000000000000000000000000000000..b1365db8fb73b93c710fd754d1c9d6148e07fbd4 --- /dev/null +++ b/plant_space_app.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import os + +import gradio as gr + +from plant.app import APP_CSS, build_app + +os.environ.setdefault("WORKBENCH_DEPLOYMENT", "space") + +demo = build_app(model_mode="openbmb") + +if __name__ == "__main__": + demo.launch( + server_port=int(os.getenv("PORT", "7860")), + share=False, + theme=gr.themes.Soft(primary_hue="green", neutral_hue="slate"), + css=APP_CSS, + mcp_server=True, + ) diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..e31ecbbc26e98356c324228addf7618d13702d43 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,42 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.E2E_BASE_URL ?? "http://127.0.0.1:7860"; +const defaultCommand = "python app.py"; +const appCommand = process.env.E2E_APP_COMMAND ?? defaultCommand; +const webServer = process.env.E2E_NO_WEBSERVER === "1" + ? undefined + : { + command: appCommand, + url: baseURL, + reuseExistingServer: true, + timeout: 120_000, + env: { + ...process.env, + PYTHONUTF8: "1", + WORKBENCH_DEPLOYMENT: process.env.WORKBENCH_DEPLOYMENT ?? "space" + } + }; + +export default defineConfig({ + testDir: "./tests/e2e", + testMatch: /workbench_.*\.spec\.ts/, + timeout: 60_000, + expect: { + timeout: 10_000 + }, + fullyParallel: false, + reporter: [["list"], ["html", { outputFolder: "assets/e2e/playwright-report" }]], + use: { + baseURL, + trace: "retain-on-failure", + video: "retain-on-failure", + screenshot: "only-on-failure" + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] } + } + ], + webServer +}); diff --git a/playwright.plant.config.ts b/playwright.plant.config.ts new file mode 100644 index 0000000000000000000000000000000000000000..b5641434480828fb95341aa8a0a83f08c5af412d --- /dev/null +++ b/playwright.plant.config.ts @@ -0,0 +1,43 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.E2E_BASE_URL ?? "http://127.0.0.1:7861"; +const defaultCommand = "python plant_space_app.py"; +const appCommand = process.env.E2E_APP_COMMAND ?? defaultCommand; +const webServer = process.env.E2E_NO_WEBSERVER === "1" + ? undefined + : { + command: appCommand, + url: baseURL, + reuseExistingServer: true, + timeout: 180_000, + env: { + ...process.env, + PORT: "7861", + PYTHONUTF8: "1", + WORKBENCH_DEPLOYMENT: "space" + } + }; + +export default defineConfig({ + testDir: "./tests/e2e", + testMatch: /plant_.*\.spec\.ts/, + timeout: 700_000, + expect: { + timeout: 30_000 + }, + fullyParallel: false, + reporter: [["list"], ["html", { outputFolder: "assets/e2e/plant-playwright-report" }]], + use: { + baseURL, + trace: "retain-on-failure", + video: "retain-on-failure", + screenshot: "only-on-failure" + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] } + } + ], + webServer +}); diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..ae53b18b88d362dba0dccce8cb423befae1a53d8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "S"] +ignore = ["S101"] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.pylint.main] +py-version = "3.11" +ignore = [".venv"] + +[tool.pylint.messages_control] +disable = [ + "missing-module-docstring", + "missing-function-docstring", + "missing-class-docstring", + "too-few-public-methods", + "too-many-arguments", + "too-many-instance-attributes", + "too-many-locals", + "too-many-positional-arguments", + "no-member", + "duplicate-code", +] + +[tool.bandit] +exclude_dirs = ["tests", ".venv"] + +[tool.coverage.run] +branch = true +source = ["app", "core", "datasets", "models", "ui"] + +[tool.coverage.report] +fail_under = 60 +show_missing = true +skip_empty = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "performance: lightweight performance checks for local scaffold behavior", +] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8df2915f6b8427284f68f3fc9865abfd64b82f8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +pytest>=8.0.0 +coverage>=7.0.0 +pytest-benchmark>=5.0.0 +ruff>=0.6.0 +mypy>=1.10.0 +pylint>=3.2.0 +bandit>=1.7.9 +pip-audit>=2.7.0 +types-PyYAML>=6.0.12 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..392561de2ad25b52e244ef80f657b061d0be2599 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +gradio>=5.38.0 +pyyaml>=6.0.0 +Pillow>=10.0.0 +huggingface_hub>=0.23.0 +requests>=2.31.0 +transformers>=5.7.0 +torch>=2.1.0 +torchvision +accelerate +datasets>=2.18.0 diff --git a/scripts/plan_hf_space.py b/scripts/plan_hf_space.py new file mode 100644 index 0000000000000000000000000000000000000000..84b9839d68d986df99951f149ae8ce4731982c70 --- /dev/null +++ b/scripts/plan_hf_space.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from deployment.hf_space import ( # noqa: E402 + create_space_commands, + hackathon_space_commands, + space_build_status, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Plan Hugging Face Space deployment") + parser.add_argument("--user", required=True, help="Hugging Face user or org") + parser.add_argument("--space", default="openbmb-local-ai-workbench") + parser.add_argument("--branch", default="main") + args = parser.parse_args() + + git_executable = shutil.which("git") or "git" + remote_output = subprocess.run( # noqa: S603 + [git_executable, "remote", "-v"], + check=False, + capture_output=True, + text=True, + ).stdout + status = space_build_status(".", remote_output) + + print("Space build status:") + print(status.as_dict()) + + print("\nCommands to run manually:") + for command in create_space_commands(args.user, args.space, args.branch): + print(f"- {command}") + + print("\nBuild Small Hackathon target Spaces:") + for name, commands in hackathon_space_commands(args.branch).items(): + print(f"\n{name}:") + for command in commands: + print(f"- {command}") + + +if __name__ == "__main__": + main() diff --git a/scripts/plan_plant_training.py b/scripts/plan_plant_training.py new file mode 100644 index 0000000000000000000000000000000000000000..b9add3c6ad010dd47efb0056d18f133f2165e165 --- /dev/null +++ b/scripts/plan_plant_training.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def main() -> None: + from plant.training import build_plant_training_plan, write_llamafactory_dataset_info + + parser = argparse.ArgumentParser(description="Create a Plant Discovery training plan.") + parser.add_argument("--config", default="plant/models.yaml") + parser.add_argument("--dataset", default="data/plant_training.jsonl") + parser.add_argument("--output-dir", default="checkpoints/plant_lora") + parser.add_argument("--adapter-repo", default="your-username/minicpm-v46-plant-lora") + parser.add_argument("--corrected-examples", type=int, default=0) + parser.add_argument( + "--write-llamafactory-info", + action="store_true", + help="Write a LLaMA-Factory dataset_info JSON preview.", + ) + args = parser.parse_args() + + plan = build_plant_training_plan( + config_path=args.config, + dataset_path=args.dataset, + output_dir=args.output_dir, + adapter_repo=args.adapter_repo, + corrected_examples=args.corrected_examples, + ) + payload = plan.to_dict() + if args.write_llamafactory_info: + payload["llamafactory_dataset_info"] = str( + write_llamafactory_dataset_info(args.dataset) + ) + print(json.dumps(payload, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_performance.ps1 b/scripts/run_performance.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..95a7160ca72567fea958cc4106603f4758322ace --- /dev/null +++ b/scripts/run_performance.ps1 @@ -0,0 +1,18 @@ +$ErrorActionPreference = "Stop" + +$python = ".venv\Scripts\python.exe" +if (-not (Test-Path $python)) { + $python = "python" + try { + & $python --version | Out-Null + } catch { + $python = "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" + } +} + +$pytestCache = "$env:TEMP\openbmb-workbench-pytest-cache" + +& $python -m pytest tests/performance -m performance -o "cache_dir=$pytestCache" +if ($LASTEXITCODE -ne 0) { + throw "Performance tests failed" +} diff --git a/scripts/run_plant_e2e.ps1 b/scripts/run_plant_e2e.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..1ef225c8c6dea5e45226ab7eb0d430908ed0948a --- /dev/null +++ b/scripts/run_plant_e2e.ps1 @@ -0,0 +1,19 @@ +$ErrorActionPreference = "Stop" + +$env:WORKBENCH_DEPLOYMENT = "space" +$env:PORT = "7861" +$env:PYTHONUTF8 = "1" +$env:PLANT_MAX_NEW_TOKENS = "320" +$env:PLANT_AUTO_THINKING = "0" + +$server = Start-Process python -ArgumentList "plant_space_app.py" -PassThru -WindowStyle Hidden +try { + Start-Sleep -Seconds 8 + $env:E2E_NO_WEBSERVER = "1" + $env:E2E_BASE_URL = "http://127.0.0.1:7861" + npx playwright test tests/e2e/plant_real_model.spec.ts --config playwright.plant.config.ts + exit $LASTEXITCODE +} +finally { + Stop-Process -Id $server.Id -ErrorAction SilentlyContinue +} diff --git a/scripts/run_playwright_e2e.ps1 b/scripts/run_playwright_e2e.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..9793a5e0fbba3c98c4229d690e77e121272a133a --- /dev/null +++ b/scripts/run_playwright_e2e.ps1 @@ -0,0 +1,7 @@ +$ErrorActionPreference = "Stop" + +if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + throw "npm is not installed. Install Node.js first, then run npm install and npm run e2e:install." +} + +npm run e2e diff --git a/scripts/run_quality.ps1 b/scripts/run_quality.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..d1fec8203bc422c514a8b6998426e2376476d0bb --- /dev/null +++ b/scripts/run_quality.ps1 @@ -0,0 +1,88 @@ +$ErrorActionPreference = "Stop" + +$python = ".venv\Scripts\python.exe" +if (-not (Test-Path $python)) { + $python = "python" + try { + & $python --version | Out-Null + } catch { + $python = "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" + } +} +$mypyCache = "$env:TEMP\openbmb-workbench-mypy-cache" +$pipAuditCache = "$env:TEMP\openbmb-workbench-pip-audit-cache" +$pytestCache = "$env:TEMP\openbmb-workbench-pytest-cache" +$coverageFile = "$env:TEMP\openbmb-workbench.coverage" + +function Invoke-Checked { + param( + [Parameter(Mandatory = $true)] + [string[]] $Command + ) + + & $Command[0] $Command[1..($Command.Length - 1)] + if ($LASTEXITCODE -ne 0) { + throw "Command failed: $($Command -join ' ')" + } +} + +Write-Host "Running tests" +Invoke-Checked @($python, "-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py") + +Write-Host "Running app smoke test" +Invoke-Checked @($python, "scripts/smoke_app.py") + +Write-Host "Running coverage" +Invoke-Checked @( + $python, + "-m", + "coverage", + "run", + "--data-file", + $coverageFile, + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_*.py" +) +Invoke-Checked @($python, "-m", "coverage", "report", "--data-file", $coverageFile) + +Write-Host "Running performance tests" +Invoke-Checked @( + $python, + "-m", + "pytest", + "tests/performance", + "-m", + "performance", + "-o", + "cache_dir=$pytestCache" +) + +Write-Host "Running ruff" +Invoke-Checked @($python, "-m", "ruff", "check", ".", "--no-cache") + +Write-Host "Running mypy" +Invoke-Checked @($python, "-m", "mypy", ".", "--cache-dir", $mypyCache) + +Write-Host "Running pylint" +Invoke-Checked @($python, "-m", "pylint", "--persistent=n", "app.py", "core", "datasets", "models", "ui") + +Write-Host "Running bandit" +Invoke-Checked @($python, "-m", "bandit", "-r", "app.py", "core", "datasets", "models", "ui") + +Write-Host "Running pip-audit" +Invoke-Checked @( + $python, + "-m", + "pip_audit", + "--cache-dir", + $pipAuditCache, + "-r", + "requirements.txt", + "-r", + "requirements-dev.txt" +) diff --git a/scripts/run_smoke.ps1 b/scripts/run_smoke.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..bce32aa71ec5db4bc559021cf3fdc9ac94e84fc9 --- /dev/null +++ b/scripts/run_smoke.ps1 @@ -0,0 +1,16 @@ +$ErrorActionPreference = "Stop" + +$python = ".venv\Scripts\python.exe" +if (-not (Test-Path $python)) { + $python = "python" + try { + & $python --version | Out-Null + } catch { + $python = "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" + } +} + +& $python scripts/smoke_app.py +if ($LASTEXITCODE -ne 0) { + throw "App smoke test failed" +} diff --git a/scripts/run_tests.ps1 b/scripts/run_tests.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..90b648e58aa6b7886cc5514d478f471549108617 --- /dev/null +++ b/scripts/run_tests.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = "Stop" + +$python = ".venv\Scripts\python.exe" +if (-not (Test-Path $python)) { + $python = "python" + try { + & $python --version | Out-Null + } catch { + $python = "$env:LOCALAPPDATA\Microsoft\WindowsApps\python3.11.exe" + } +} +$coverageFile = "$env:TEMP\openbmb-workbench.coverage" + +& $python -m coverage run --data-file $coverageFile -m unittest discover -s tests -p "test_*.py" +if ($LASTEXITCODE -ne 0) { + throw "Tests failed" +} +& $python -m coverage report --data-file $coverageFile +if ($LASTEXITCODE -ne 0) { + throw "Coverage failed" +} diff --git a/scripts/smoke_app.py b/scripts/smoke_app.py new file mode 100644 index 0000000000000000000000000000000000000000..b108aad7a8434f1950ee5bcbd107ce472da69fd2 --- /dev/null +++ b/scripts/smoke_app.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app import build_app + + +def main() -> None: + demo = build_app() + try: + print(type(demo).__name__) + finally: + demo.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_structure.ps1 b/scripts/verify_structure.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..3472abd615f26252a23e9f0955a5942afb601572 --- /dev/null +++ b/scripts/verify_structure.ps1 @@ -0,0 +1,58 @@ +$ErrorActionPreference = "Stop" + +$required = @( + "AGENTS.md", + "README.md", + "app.py", + "requirements.txt", + "config/models.yaml", + "config/training.yaml", + "docs/README.md", + "docs/TASKS.md", + "docs/IMPLEMENTATION_STATUS.md", + "docs/ACCEPTANCE_CRITERIA.md", + "docs/ROADMAP.md", + "docs/USAGE.md", + "docs/ARCHITECTURE.md", + "docs/EXTENDING.md", + "core/events.py", + "core/registry.py", + "models/model_catalog.py", + "models/response_parsing.py", + "models/base.py", + "models/llama_cpp_python_service.py", + "models/llama_cpp_service.py", + "models/ollama_service.py", + "models/placeholder_service.py", + "models/service_factory.py", + "datasets/field_notes.py", + "datasets/loader.py", + "scripts/run_smoke.ps1", + "scripts/smoke_app.py", + "ui/chat_tab.py", + "ui/vision_tab.py", + "ui/dataset_tab.py", + "ui/train_tab.py", + "ui/export_tab.py", + "ui/notes_tab.py", + "ui/traces_tab.py", + "ui/agent_tab.py", + "ui/status_tab.py" +) + +$missing = @() +foreach ($path in $required) { + if (-not (Test-Path $path)) { + $missing += $path + } +} + +if ($missing.Count -gt 0) { + Write-Host "Missing required files:" + foreach ($path in $missing) { + Write-Host " - $path" + } + exit 1 +} + +Write-Host "Structure check passed. $($required.Count) required files found." diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/e2e/plant_real_model.spec.ts b/tests/e2e/plant_real_model.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a39aa7fcd6222f5f10b072a0f60bc8dcf0d97105 --- /dev/null +++ b/tests/e2e/plant_real_model.spec.ts @@ -0,0 +1,104 @@ +import { expect, type Page, test } from "@playwright/test"; +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +type StoryStep = { + title: string; + description: string; + fileName: string; +}; + +const assetsDir = path.join(process.cwd(), "assets", "e2e", "plant"); +const sampleImage = path.join(process.cwd(), "assets", "plant_sample.jpg"); +const runRealInference = process.env.RUN_REAL_MODEL_E2E === "1"; + +test.describe("Plant Identification Tool real-model journey", () => { + test("captures the Plant tool in OpenBMB mode", async ({ page }) => { + test.slow(); + mkdirSync(assetsDir, { recursive: true }); + const breadcrumbs: string[] = []; + const mark = (step: string): void => { + breadcrumbs.push(`${new Date().toISOString()} ${step}`); + writeFileSync(path.join(assetsDir, "real-inference-breadcrumbs.txt"), `${breadcrumbs.join("\n")}\n`); + }; + + mark("goto"); + await page.goto("/", { waitUntil: "domcontentloaded" }); + await expect(page.getByRole("heading", { name: /Plant Discovery/i })).toBeVisible(); + await expect(page.getByText(/Local-first reference app/i)).toBeVisible(); + + const steps: StoryStep[] = []; + await capture(page, "01-plant-home.png", "Plant Tool Home", "The Plant Space opens in OpenBMB real-model mode.", steps); + mark("home-captured"); + + await selectTab(page, "Identify"); + await page.getByLabel("Analysis mode").selectOption("standard").catch(async () => { + await page.getByLabel("Analysis mode").click(); + await page.getByRole("option", { name: /^standard$/i }).click(); + }); + + if (runRealInference) { + mark("upload-start"); + await page.locator("input[type='file']").first().setInputFiles(sampleImage); + mark("upload-done"); + await page.getByRole("button", { name: /^Identify plant$/i }).click(); + mark("identify-clicked"); + await expect( + page.getByRole("heading", { name: /Lady's Mantle|Model unavailable/i }).first() + ).toBeVisible({ timeout: 180_000 }); + mark("result-visible"); + } + await capture(page, "02-identify-real-model.png", "Real Model Identify", "A public-safe image is ready for MiniCPM-V identification.", steps); + + await selectTab(page, "Corrections"); + await expect(page.getByRole("button", { name: /^Export training JSONL$/i })).toBeVisible(); + await capture(page, "03-corrections-export.png", "Corrections Export", "Corrections export remains available for real model outputs.", steps); + + await selectTab(page, "Stats"); + await expect(page.getByRole("button", { name: /^Refresh stats$/i })).toBeVisible(); + await page.getByRole("button", { name: /^Refresh stats$/i }).click(); + await capture(page, "04-plant-stats.png", "Plant Stats", "The Plant app reports species and correction counts.", steps); + + writeFileSync(path.join(assetsDir, "user-story.md"), renderStoryMarkdown(steps)); + }); +}); + +async function selectTab(page: Page, name: string): Promise { + const tab = page.getByRole("tab", { name: new RegExp(`^${name}$`, "i") }); + await expect(tab).toBeVisible(); + await tab.click(); + await expect(tab).toHaveAttribute("aria-selected", "true"); +} + +async function capture( + page: Page, + fileName: string, + title: string, + description: string, + steps: StoryStep[], +): Promise { + await page.screenshot({ path: path.join(assetsDir, fileName), fullPage: true }); + steps.push({ title, description, fileName }); +} + +function renderStoryMarkdown(steps: StoryStep[]): string { + const lines = [ + "# Plant Identification Tool Screenshots", + "", + "These screenshots use the OpenBMB-mode Plant Space entrypoint. Real model inference is executed only when `RUN_REAL_MODEL_E2E=1` is set.", + "" + ]; + + for (const [index, step] of steps.entries()) { + lines.push( + `## ${index + 1}. ${step.title}`, + "", + step.description, + "", + `![${step.title}](./${step.fileName})`, + "" + ); + } + + return `${lines.join("\n")}\n`; +} diff --git a/tests/e2e/workbench_real_model.spec.ts b/tests/e2e/workbench_real_model.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..3917c5fe979cd8496c239ba2f77e6fc0f0ef4971 --- /dev/null +++ b/tests/e2e/workbench_real_model.spec.ts @@ -0,0 +1,134 @@ +import { expect, type Locator, type Page, test } from "@playwright/test"; +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +type StoryStep = { + title: string; + description: string; + fileName: string; +}; + +const assetsDir = path.join(process.cwd(), "assets", "e2e", "workbench"); +const sampleDataset = path.join(process.cwd(), "data", "public_sample_field_notes.csv"); + +test.describe("OpenBMB Workbench real-backend journey", () => { + test("captures the Workbench with real backend choices and public sample data", async ({ page }) => { + test.slow(); + mkdirSync(assetsDir, { recursive: true }); + + await page.goto("/"); + await expect(page.getByRole("heading", { name: /OpenBMB Local AI Workbench/i })).toBeVisible(); + + const steps: StoryStep[] = []; + await capture(page, "01-workbench-home.png", "Workbench Home", "The Workbench opens with real-backend setup visible.", steps); + + await selectTab(page, "Chat"); + await expect(page.getByLabel("Backend").filter({ visible: true }).first()).not.toContainText("placeholder"); + await selectDropdown(page, "Backend", "llama-cpp-python"); + await fillVisibleTextbox(page, 0, "You are a concise local model assistant."); + await fillVisibleTextbox(page, 1, "Reply in one short sentence that local GGUF inference is working."); + await clickButton(page, "Run"); + await expect(page.getByText(/Chat response generated/i)).toBeVisible({ timeout: 180_000 }); + const chatResponse = page.getByRole("textbox").filter({ visible: true }).nth(2); + await expect(chatResponse).toHaveValue(/.+/, { timeout: 10_000 }); + await expect(chatResponse).not.toHaveValue(/\[(llama-cpp-python unavailable|llama\.cpp unavailable|Transformers unavailable)\]/i); + await capture(page, "02-chat-real-backend-response.png", "Real Text Response", "Chat generated a visible answer through the local GGUF llama-cpp-python backend.", steps); + + await selectTab(page, "Vision"); + await expect(page.getByLabel("Backend").filter({ visible: true }).first()).not.toContainText("placeholder"); + await selectDropdown(page, "Backend", "transformers"); + await fillByLabel(page, "Prompt", "Describe the plant image for a field note."); + await capture(page, "03-vision-real-backend.png", "Real Vision Backend", "Vision is configured for a real MiniCPM-V Transformers backend.", steps); + + await selectTab(page, "Dataset"); + await fillByLabel(page, "Dataset ID or local path", sampleDataset); + await fillByLabel(page, "Split", "train"); + await clickButton(page, "Preview dataset"); + await expect(page.getByText(/Status: Local dataset preview loaded\./i)).toBeVisible(); + await capture(page, "04-public-sample-data.png", "Public Sample Data", "The Workbench previews committed public-safe sample corrections.", steps); + + await selectTab(page, "Status"); + await expect(page.getByText(/Model and backend status/i)).toBeVisible(); + await capture(page, "05-backend-status.png", "Backend Status", "Status lists OpenBMB models and real local backend readiness.", steps); + + writeFileSync(path.join(assetsDir, "user-story.md"), renderStoryMarkdown(steps)); + }); +}); + +async function capture( + page: Page, + fileName: string, + title: string, + description: string, + steps: StoryStep[], +): Promise { + await page.screenshot({ path: path.join(assetsDir, fileName), fullPage: true }); + steps.push({ title, description, fileName }); +} + +async function selectTab(page: Page, name: string): Promise { + const tab = page.getByRole("tab", { name: exactText(name) }); + await expect(tab).toBeVisible(); + await tab.click(); + await expect(tab).toHaveAttribute("aria-selected", "true"); +} + +async function selectDropdown(page: Page, label: string, value: string): Promise { + const field = page.getByLabel(exactText(label)).filter({ visible: true }).first(); + await field.selectOption(value).catch(async () => { + await field.click(); + await page.getByRole("option", { name: exactText(value) }).click(); + }); +} + +async function fillByLabel(page: Page, label: string, value: string): Promise { + const field = page.getByLabel(label).filter({ visible: true }).first(); + await expect(field, `Expected editable field labelled "${label}"`).toBeVisible(); + await field.fill(value); +} + +async function fillVisibleTextbox(page: Page, index: number, value: string): Promise { + const field = page.getByRole("textbox").filter({ visible: true }).nth(index); + await expect(field, `Expected visible textbox at index ${index}`).toBeVisible(); + await field.fill(value); +} + +async function clickButton(page: Page, name: string): Promise { + const button = visibleButton(page, name); + await expect(button, `Expected button "${name}"`).toBeVisible(); + await button.click(); +} + +function visibleButton(page: Page, name: string): Locator { + return page.getByRole("button", { name: exactText(name) }).filter({ visible: true }).first(); +} + +function exactText(text: string): RegExp { + return new RegExp(`^${escapeRegex(text)}$`, "i"); +} + +function escapeRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function renderStoryMarkdown(steps: StoryStep[]): string { + const lines = [ + "# Workbench Real-Backend Screenshots", + "", + "These screenshots are generated without mock responses. The chat screenshot runs the local GGUF llama-cpp-python backend and captures the model response.", + "" + ]; + + for (const [index, step] of steps.entries()) { + lines.push( + `## ${index + 1}. ${step.title}`, + "", + step.description, + "", + `![${step.title}](./${step.fileName})`, + "" + ); + } + + return `${lines.join("\n")}\n`; +} diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/tests/performance/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/performance/test_scaffold_performance.py b/tests/performance/test_scaffold_performance.py new file mode 100644 index 0000000000000000000000000000000000000000..ec8aff344001a91f216b3d1b84d96f872dc9b7f6 --- /dev/null +++ b/tests/performance/test_scaffold_performance.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import time + +import pytest + +from models.model_catalog import load_model_catalog, validate_catalog +from models.placeholder_service import PlaceholderModelService + + +@pytest.mark.performance +def test_catalog_load_and_validate_is_fast_enough_for_app_start() -> None: + start = time.perf_counter() + + catalog = load_model_catalog("config/models.yaml") + validate_catalog(catalog) + + elapsed = time.perf_counter() - start + assert elapsed < 0.25 + + +@pytest.mark.performance +def test_placeholder_response_is_fast_enough_for_ui_smoke_flow() -> None: + catalog = load_model_catalog("config/models.yaml") + service = PlaceholderModelService(catalog["minicpm5_1b"]) + + start = time.perf_counter() + response = service.chat("Be concise.", "Smoke test") + elapsed = time.perf_counter() - start + + assert "Real inference is not wired yet" in response + assert elapsed < 0.05 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/unit/test_agent_runner.py b/tests/unit/test_agent_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..92d7527e4453d094d79b446ad072830f19e927d8 --- /dev/null +++ b/tests/unit/test_agent_runner.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from agent.runner import ( + AGENT_SYSTEM_PROMPT, + default_safety_gates, + export_agent_traces, + export_agent_traces_hf_dataset, + run_agent_loop, + run_paper_to_code_loop, + save_agent_trace, +) + + +class AgentRunnerTest(unittest.TestCase): + def test_agent_loop_creates_research_plan_implementation_trace(self) -> None: + session = run_agent_loop("Improve dataset stats") + + phases = [step.phase for step in session.steps] + self.assertIn("research", phases) + self.assertIn("plan", phases) + self.assertIn("implement", phases) + self.assertIn("verify", phases) + self.assertIn("safe_calculator", session.tools) + self.assertTrue(session.safety_gates) + self.assertEqual(session.as_dict()["system_prompt"], AGENT_SYSTEM_PROMPT) + + def test_agent_loop_can_use_safe_calculator_tool(self) -> None: + session = run_agent_loop("calculate: 2 + 2") + + self.assertIn("tool:safe_calculator", [step.phase for step in session.steps]) + self.assertIn("4.0", session.as_markdown()) + + def test_saves_and_exports_agent_trace(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "agent_traces.jsonl" + output = Path(tmp) / "exports" / "agent_traces.jsonl" + session = run_agent_loop("Improve docs") + + saved = save_agent_trace(session, source) + exported = export_agent_traces(source, output) + + self.assertEqual(saved, source) + self.assertEqual(exported, output) + self.assertIn("Improve docs", output.read_text(encoding="utf-8")) + + def test_exports_agent_trace_dataset(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "agent_traces.jsonl" + target = Path(tmp) / "dataset" + save_agent_trace(run_agent_loop("Trace me"), source) + + exported = export_agent_traces_hf_dataset(source, target) + + self.assertEqual(exported, target) + self.assertTrue((target / "data.jsonl").exists()) + self.assertTrue((target / "README.md").exists()) + + def test_paper_to_code_trace_has_required_phases_and_safety_gates(self) -> None: + session = run_paper_to_code_loop( + "Demo paper", + "Claims a local reward model can rank completions.", + "Implement deterministic reward eval.", + ) + + phases = [step.phase for step in session.steps] + self.assertEqual(phases, ["research", "plan", "implement", "verify"]) + self.assertIn("Paper-to-code", session.task) + self.assertIn("No shell commands", session.as_markdown()) + + def test_default_safety_gates_block_external_side_effects(self) -> None: + gates = default_safety_gates() + + self.assertTrue(any("downloaded automatically" in gate for gate in gates)) + self.assertTrue(any("matching test" in gate for gate in gates)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_app_shell.py b/tests/unit/test_app_shell.py new file mode 100644 index 0000000000000000000000000000000000000000..00fa4f763113ba0ae60bdd5f6a2bda9ca3ba5a4e --- /dev/null +++ b/tests/unit/test_app_shell.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +from app import APP_CSS + + +class AppShellTest(unittest.TestCase): + def test_app_css_includes_compact_responsive_rules(self) -> None: + self.assertIn("@media (max-width: 720px)", APP_CSS) + self.assertIn("max-width: 1180px", APP_CSS) + self.assertIn("overflow-x: auto", APP_CSS) + self.assertIn("min-height: 2.5rem", APP_CSS) + + def test_app_launch_enables_gradio_mcp_server(self) -> None: + source = Path("app.py").read_text(encoding="utf-8") + + self.assertIn("mcp_server=True", source) + + def test_app_disables_gradio_analytics_for_local_first_launch(self) -> None: + source = Path("app.py").read_text(encoding="utf-8") + + self.assertIn("analytics_enabled=False", source) + + def test_app_includes_vllm_tab(self) -> None: + source = Path("app.py").read_text(encoding="utf-8") + + self.assertIn('with gr.Tab("vLLM")', source) + self.assertIn("build_vllm_tab(catalog)", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py new file mode 100644 index 0000000000000000000000000000000000000000..fa2e1a99809af52eec5a44f6051e1a2334dccb79 --- /dev/null +++ b/tests/unit/test_core.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import unittest + +from core.app_state import AppState +from core.events import Event, EventBus, EventType +from core.registry import Registry + + +class CoreTest(unittest.TestCase): + def test_event_bus_calls_registered_handler(self) -> None: + bus = EventBus() + seen = [] + + @bus.on(EventType.DATASET_LOADED) + def handler(event: Event) -> None: + seen.append(event.payload["rows"]) + + bus.emit(Event(EventType.DATASET_LOADED, {"rows": 3})) + + self.assertEqual(seen, [3]) + + def test_event_type_behaves_like_string_on_python_310(self) -> None: + self.assertIsInstance(EventType.DATASET_LOADED, str) + self.assertEqual(str(EventType.DATASET_LOADED), "dataset_loaded") + self.assertEqual(EventType.DATASET_LOADED.value, "dataset_loaded") + + def test_registry_registers_and_gets_items(self) -> None: + registry: Registry[int] = Registry() + registry.register("answer", 42) + + self.assertEqual(registry.get("answer"), 42) + self.assertEqual(registry.list(), ["answer"]) + + def test_registry_raises_for_missing_item(self) -> None: + registry: Registry[int] = Registry() + + with self.assertRaises(KeyError): + registry.get("missing") + + def test_app_state_records_and_dispatches_events(self) -> None: + bus = EventBus() + state = AppState(bus) + seen = [] + + @bus.on(EventType.FIELD_NOTE_SAVED) + def handler(event: Event) -> None: + seen.append(event.payload["model_id"]) + + state.emit(Event(EventType.FIELD_NOTE_SAVED, {"model_id": "minicpm"})) + + self.assertEqual(seen, ["minicpm"]) + self.assertEqual(state.recent_events()[0]["payload"]["model_id"], "minicpm") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_dataset_loader.py b/tests/unit/test_dataset_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c630e2d3da583bc931ef38d3d965fd4a44fe01 --- /dev/null +++ b/tests/unit/test_dataset_loader.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from datasets.loader import ( + dataset_statistics, + preview_huggingface_dataset, + preview_local_dataset, +) + + +class DatasetLoaderTest(unittest.TestCase): + def test_previews_csv(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.csv" + path.write_text("prompt,response\nhello,world\nfoo,bar\n", encoding="utf-8") + + preview = preview_local_dataset(path) + + self.assertEqual(preview.rows, 2) + self.assertEqual(preview.columns, ["prompt", "response"]) + self.assertEqual(preview.samples[0]["prompt"], "hello") + + def test_previews_jsonl(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.jsonl" + path.write_text('{"prompt":"hello","response":"world"}\n', encoding="utf-8") + + preview = preview_local_dataset(path) + + self.assertEqual(preview.rows, 1) + self.assertEqual(preview.columns, ["prompt", "response"]) + self.assertIn("sample_1", preview.as_table()[3][0]) + + def test_rejects_jsonl_arrays(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.jsonl" + path.write_text('["not", "an", "object"]\n', encoding="utf-8") + + with self.assertRaises(ValueError): + preview_local_dataset(path) + + def test_rejects_unsupported_format(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.txt" + path.write_text("hello", encoding="utf-8") + + with self.assertRaises(ValueError): + preview_local_dataset(path) + + def test_calculates_dataset_statistics(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.csv" + path.write_text("prompt,response\nhello,world\nempty,\n", encoding="utf-8") + + stats = dataset_statistics(path) + + self.assertEqual(stats.rows, 2) + self.assertEqual(stats.columns, 2) + self.assertEqual(stats.non_empty_by_column["response"], 1) + + def test_calculates_jsonl_dataset_statistics(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.jsonl" + path.write_text( + '{"prompt":"hello","response":"world"}\n' + '{"prompt":"empty","response":""}\n', + encoding="utf-8", + ) + + stats = dataset_statistics(path) + + self.assertEqual(stats.as_dict()["rows"], 2) + self.assertEqual(stats.non_empty_by_column["response"], 1) + + def test_huggingface_preview_reports_missing_optional_dependency(self) -> None: + preview = preview_huggingface_dataset("demo/dataset") + + self.assertEqual(preview.dataset_id, "demo/dataset") + self.assertIn("datasets", preview.status) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_deployment_policy.py b/tests/unit/test_deployment_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..825171809cb6a28dabbc4347879cd2c1de2a8b0e --- /dev/null +++ b/tests/unit/test_deployment_policy.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import unittest + +from core.deployment import ( + DeploymentPolicy, + default_backend_for_policy, + deployment_mode, + ensure_backend_allowed, + ensure_demo_mode_allowed, + filter_backends_for_policy, +) + + +class DeploymentPolicyTest(unittest.TestCase): + def test_default_deployment_mode_is_local(self) -> None: + self.assertEqual(deployment_mode(""), "local") + + def test_rejects_unknown_deployment_mode(self) -> None: + with self.assertRaises(ValueError): + deployment_mode("prod") + + def test_space_mode_filters_placeholder_backend(self) -> None: + policy = DeploymentPolicy("space") + + backends = filter_backends_for_policy(["placeholder", "transformers"], policy) + + self.assertEqual(backends, ["transformers"]) + + def test_space_mode_refuses_placeholder_backend(self) -> None: + with self.assertRaises(ValueError): + ensure_backend_allowed("placeholder", DeploymentPolicy("space")) + + def test_space_mode_refuses_demo_mode(self) -> None: + with self.assertRaises(ValueError): + ensure_demo_mode_allowed(DeploymentPolicy("space")) + + def test_space_mode_prefers_real_backend(self) -> None: + backend = default_backend_for_policy( + ["placeholder", "transformers"], + "placeholder", + DeploymentPolicy("space"), + ) + + self.assertEqual(backend, "transformers") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_evaluation.py b/tests/unit/test_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..55cb42bf329e549141e1d26a6f37c20eefcb59db --- /dev/null +++ b/tests/unit/test_evaluation.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from training.evaluation import ( + attach_perplexity, + compare_base_vs_tuned, + default_prompt_cases, + evaluate_responses, + load_prompt_cases, + log_eval_report, + perplexity_from_losses, +) + + +class EvaluationTest(unittest.TestCase): + def test_evaluates_exact_match_rate(self) -> None: + cases = default_prompt_cases() + + report = evaluate_responses(cases, ["field note", "wrong", "no"]) + + self.assertEqual(report.exact_match_rate, 2 / 3) + self.assertEqual(report.rows[1].notes, "review") + + def test_compares_base_vs_tuned_reports(self) -> None: + cases = default_prompt_cases() + base = evaluate_responses(cases, ["wrong", "wrong", "wrong"]) + tuned = evaluate_responses(cases, ["field note", "jsonl", "no"]) + + comparison = compare_base_vs_tuned(base, tuned) + + self.assertEqual(comparison.delta, 1.0) + + def test_loads_prompt_cases_from_jsonl(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "cases.jsonl" + path.write_text( + json.dumps({"prompt": "Prompt", "expected": "Answer"}) + "\n", + encoding="utf-8", + ) + + cases = load_prompt_cases(path) + + self.assertEqual(cases[0].prompt, "Prompt") + self.assertEqual(cases[0].expected, "Answer") + + def test_logs_eval_report(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "eval.jsonl" + report = evaluate_responses(default_prompt_cases(), ["field note"]) + + saved = log_eval_report(report, path) + + self.assertEqual(saved, path) + self.assertIn("exact_match_rate", path.read_text(encoding="utf-8")) + + def test_calculates_perplexity_from_average_loss(self) -> None: + perplexity = perplexity_from_losses([0.0, 0.0]) + + self.assertEqual(perplexity, 1.0) + self.assertIsNone(perplexity_from_losses([])) + + def test_attaches_perplexity_to_eval_report(self) -> None: + report = evaluate_responses(default_prompt_cases(), ["field note"]) + + updated = attach_perplexity(report, [0.0]) + + self.assertEqual(updated.exact_match_rate, report.exact_match_rate) + self.assertEqual(updated.perplexity, 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_export_plan.py b/tests/unit/test_export_plan.py new file mode 100644 index 0000000000000000000000000000000000000000..7bc94e09244e323bcf2b39b83c781ad84a986cde --- /dev/null +++ b/tests/unit/test_export_plan.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from models.model_catalog import load_model_catalog +from training.export import ( + ToolStatus, + build_export_plan, + detect_llama_cpp_tools, + list_exported_files, +) +from ui.export_tab import export_download_paths + + +class ExportPlanTest(unittest.TestCase): + def test_detects_llama_cpp_tools_without_requiring_install(self) -> None: + statuses = detect_llama_cpp_tools(which_func=lambda name: f"C:/tools/{name}.exe") + + self.assertEqual( + [status.name for status in statuses], + ["llama-server", "llama-cli", "llama-quantize"], + ) + self.assertTrue(all(status.available for status in statuses)) + + def test_builds_non_executing_official_gguf_plan(self) -> None: + catalog = load_model_catalog("config/models.yaml") + tools = [ToolStatus("llama-quantize", False, "")] + + plan = build_export_plan(catalog["minicpm_v46"], "Q4_K_M", "exports", tools) + data = plan.as_dict() + + self.assertEqual(data["official_gguf_repo"], "openbmb/MiniCPM-V-4.6-gguf") + self.assertIn("huggingface-cli", data["download_command"]) + self.assertIn("llama-quantize", data["quantize_command"]) + self.assertFalse(data["executes_commands"]) + self.assertFalse(data["startup_downloads"]) + + def test_rejects_unknown_quantization(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + with self.assertRaises(ValueError): + build_export_plan(catalog["minicpm5_1b"], "Q2_UNKNOWN") + + def test_lists_exported_files(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) + file_path = output / "model.gguf" + file_path.write_bytes(b"gguf") + + rows = list_exported_files(output) + + self.assertEqual(rows, [[str(file_path), "4"]]) + + def test_export_download_paths_prioritizes_existing_planned_files(self) -> None: + catalog = load_model_catalog("config/models.yaml") + tools = [ToolStatus("llama-quantize", False, "")] + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + model_dir = root / "minicpm_v46" + model_dir.mkdir() + extra_file = root / "notes.txt" + extra_file.write_text("manual note", encoding="utf-8") + + plan = build_export_plan(catalog["minicpm_v46"], "Q4_K_M", root, tools) + planned_file = model_dir / plan.official_gguf_file + planned_file.write_bytes(b"planned gguf") + + rows = list_exported_files(root) + paths = export_download_paths(plan.as_dict(), rows) + + self.assertEqual(paths[0], str(planned_file)) + self.assertIn(str(extra_file), paths) + + def test_export_download_paths_omits_missing_planned_files(self) -> None: + catalog = load_model_catalog("config/models.yaml") + tools = [ToolStatus("llama-quantize", False, "")] + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + extra_file = root / "already-exported.gguf" + extra_file.write_bytes(b"existing") + + plan = build_export_plan(catalog["minicpm_v46"], "Q4_K_M", root, tools) + paths = export_download_paths(plan.as_dict(), list_exported_files(root)) + + self.assertEqual(paths, [str(extra_file)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_field_notes.py b/tests/unit/test_field_notes.py new file mode 100644 index 0000000000000000000000000000000000000000..779b1b30b991132a2ccb2604f2e22f321da06712 --- /dev/null +++ b/tests/unit/test_field_notes.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import csv +import tempfile +import unittest +from pathlib import Path + +from datasets.field_notes import FieldNote, FieldNoteStore, SQLiteFieldNoteStore + + +class FieldNoteStoreTest(unittest.TestCase): + def test_save_writes_correction_row(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "field_notes.csv" + store = FieldNoteStore(path) + + saved_path = store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="What plant is this?", + response="Unknown plant", + correction="It is basil.", + tags="plant-id,demo", + image_path="images/basil.jpg", + video_path="", + use_for_training=True, + ) + ) + + self.assertEqual(saved_path, path) + with path.open(encoding="utf-8", newline="") as f: + rows = list(csv.DictReader(f)) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["model_id"], "minicpm5_1b") + self.assertEqual(rows[0]["correction"], "It is basil.") + self.assertEqual(rows[0]["image_path"], "images/basil.jpg") + self.assertEqual(rows[0]["use_for_training"], "True") + + def test_created_at_uses_utc_offset_without_python_311_utc_import(self) -> None: + note = FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt", + response="Response", + correction="Correction", + tags="demo", + ) + + self.assertTrue(note.created_at.endswith("+00:00")) + + def test_exports_corrected_notes_to_jsonl(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "field_notes.csv" + output = Path(tmp) / "field_notes.jsonl" + store = FieldNoteStore(path) + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt", + response="Response", + correction="Correction", + tags="demo", + ) + ) + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt 2", + response="Response 2", + correction="", + tags="demo", + ) + ) + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt 3", + response="Response 3", + correction="Correction 3", + tags="demo", + use_for_training=False, + ) + ) + + exported = store.export_jsonl(output) + + self.assertEqual(exported, output) + lines = output.read_text(encoding="utf-8").splitlines() + self.assertEqual(len(lines), 1) + self.assertIn("Correction", lines[0]) + + def test_filters_notes_by_tag_and_training_flag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = FieldNoteStore(Path(tmp) / "field_notes.csv") + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt", + response="Response", + correction="Correction", + tags="ocr,plant-id", + use_for_training=True, + ) + ) + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt 2", + response="Response 2", + correction="Correction 2", + tags="audio", + use_for_training=False, + ) + ) + + self.assertEqual(len(store.list_notes(tag="plant-id")), 1) + self.assertEqual(len(store.list_notes(training_only=True)), 1) + + def test_exports_local_hf_dataset_directory(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = FieldNoteStore(Path(tmp) / "field_notes.csv") + target = Path(tmp) / "hf_dataset" + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt", + response="Response", + correction="Correction", + tags="demo", + ) + ) + + exported = store.export_hf_dataset(target) + + self.assertEqual(exported, target) + self.assertTrue((target / "data.jsonl").exists()) + self.assertTrue((target / "README.md").exists()) + + def test_sqlite_store_saves_and_lists_notes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = SQLiteFieldNoteStore(Path(tmp) / "field_notes.sqlite") + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Prompt", + response="Response", + correction="Correction", + tags="demo", + image_path="image.png", + use_for_training=True, + ) + ) + + notes = store.list_notes(corrected_only=True, tag="demo", training_only=True) + + self.assertEqual(len(notes), 1) + self.assertEqual(notes[0].image_path, "image.png") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_gitignore_policy.py b/tests/unit/test_gitignore_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..9e2c534c354132d168b8a987bd1e242f2b48274e --- /dev/null +++ b/tests/unit/test_gitignore_policy.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +class GitignorePolicyTest(unittest.TestCase): + def test_gitignore_blocks_secrets_caches_and_model_weights(self) -> None: + patterns = set(Path(".gitignore").read_text(encoding="utf-8").splitlines()) + + required = { + ".env", + ".env.*", + "*.key", + "*.pem", + "data/*", + "exports/*", + "models_cache/", + "hf_cache/", + "wandb/", + "*.gguf", + "*.safetensors", + "*.bin", + "*.ckpt", + "*.onnx", + "*.tflite", + "*.pt", + "*.pth", + "*.model", + } + + self.assertTrue(required.issubset(patterns)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_hf_space_deploy.py b/tests/unit/test_hf_space_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..7393df564c051a1504421fc84d54dc1bbb08b922 --- /dev/null +++ b/tests/unit/test_hf_space_deploy.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from deployment.hf_space import ( + create_space_commands, + hackathon_space_commands, + has_space_remote, + missing_required_files, + readme_has_space_metadata, + space_build_status, +) + + +class HuggingFaceSpaceDeployTest(unittest.TestCase): + def test_creates_manual_space_commands(self) -> None: + commands = create_space_commands("demo-user", "demo-space") + + self.assertEqual(commands[0], "hf auth login") + self.assertIn( + "repo create demo-user/demo-space --type space --space-sdk gradio", + commands[1], + ) + self.assertIn("https://huggingface.co/spaces/demo-user/demo-space", commands[2]) + self.assertEqual(commands[3], "git push space main") + self.assertIn("app.py", commands[4]) + + def test_requires_hf_user(self) -> None: + with self.assertRaises(ValueError): + create_space_commands("") + + def test_detects_space_remote(self) -> None: + remotes = ( + "origin\thttps://github.com/Ckal/codex.git (fetch)\n" + "space-workbench\thttps://huggingface.co/spaces/u/s (push)" + ) + + self.assertTrue(has_space_remote(remotes, "space-workbench")) + + def test_hackathon_space_commands_define_two_remotes(self) -> None: + commands = hackathon_space_commands() + + self.assertIn("workbench", commands) + self.assertIn("plant", commands) + self.assertIn("space-workbench", commands["workbench"][2]) + self.assertIn("space-plant", commands["plant"][2]) + self.assertIn("plant_space_app.py", commands["plant"][4]) + + def test_validates_space_build_shape(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "config").mkdir() + for path in [ + "app.py", + "plant_space_app.py", + "requirements.txt", + "config/models.yaml", + "config/training.yaml", + ]: + (root / path).write_text("", encoding="utf-8") + (root / "README.md").write_text( + "---\ntitle: Demo\nsdk: gradio\napp_file: app.py\n---\n", + encoding="utf-8", + ) + + status = space_build_status(root, "space\thttps://huggingface.co/spaces/u/s (push)") + + self.assertEqual(missing_required_files(root), []) + self.assertTrue(readme_has_space_metadata(root)) + self.assertTrue(status.required_files_present) + self.assertTrue(status.has_space_remote) + + def test_plan_script_runs_from_script_path(self) -> None: + result = subprocess.run( # noqa: S603 + [sys.executable, "scripts/plan_hf_space.py", "--user", "demo-user"], + check=True, + capture_output=True, + text=True, + ) + + self.assertIn("Commands to run manually", result.stdout) + self.assertIn("hf auth login", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_llama_cpp_python_service.py b/tests/unit/test_llama_cpp_python_service.py new file mode 100644 index 0000000000000000000000000000000000000000..67b11dcad01d04f88edbe3e0dfdbe8277f57c0c4 --- /dev/null +++ b/tests/unit/test_llama_cpp_python_service.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest + +from models.llama_cpp_python_service import LlamaCppPythonService +from models.model_catalog import load_model_catalog +from models.service_factory import create_text_service + + +class LlamaCppPythonServiceTest(unittest.TestCase): + def test_factory_creates_llama_cpp_python_service(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + service = create_text_service(catalog["minicpm5_1b"], "llama-cpp-python") + + self.assertIsInstance(service, LlamaCppPythonService) + + def test_status_reports_missing_package_or_missing_model(self) -> None: + status = LlamaCppPythonService.status() + + self.assertFalse(status.available) + self.assertTrue( + "not installed" in status.detail or "no GGUF model path" in status.detail + ) + + def test_chat_returns_clear_unavailable_message(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + response = LlamaCppPythonService(catalog["minicpm5_1b"]).chat("", "Hello") + + self.assertIn("[llama-cpp-python unavailable]", response) + + def test_extracts_chat_completion_response(self) -> None: + response = LlamaCppPythonService._extract_response( + {"choices": [{"message": {"content": "hello"}}]} + ) + + self.assertEqual(response, "hello") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_llama_cpp_service.py b/tests/unit/test_llama_cpp_service.py new file mode 100644 index 0000000000000000000000000000000000000000..15d62d60ac9e48ecf706af8622521c1b421b5d22 --- /dev/null +++ b/tests/unit/test_llama_cpp_service.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import unittest + +import requests + +from models.llama_cpp_service import LlamaCppConfig, LlamaCppService, local_file_status +from models.model_catalog import load_model_catalog +from models.service_factory import create_text_service + + +class LlamaCppServiceTest(unittest.TestCase): + def test_factory_creates_llama_cpp_service(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + service = create_text_service(catalog["minicpm5_1b"], "llama.cpp") + + self.assertIsInstance(service, LlamaCppService) + + def test_status_reports_missing_executable(self) -> None: + status = LlamaCppService.status(which_func=lambda _name: None) + + self.assertFalse(status.available) + self.assertIn("not found", status.detail) + + def test_launch_command_includes_model_and_mmproj(self) -> None: + catalog = load_model_catalog("config/models.yaml") + service = LlamaCppService( + catalog["minicpm_v46"], + LlamaCppConfig( + server_path="C:/llama/llama-server.exe", + model_path="model.gguf", + mmproj_path="mmproj.gguf", + ), + ) + + self.assertEqual( + service.launch_command(), + [ + "C:/llama/llama-server.exe", + "-m", + "model.gguf", + "--mmproj", + "mmproj.gguf", + ], + ) + + def test_launch_command_is_empty_without_model_path(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + service = LlamaCppService(catalog["minicpm5_1b"]) + + self.assertEqual(service.launch_command(), []) + + def test_chat_returns_clear_unavailable_message(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + response = LlamaCppService(catalog["minicpm5_1b"]).chat("", "Hello") + + self.assertIn("[llama.cpp unavailable]", response) + + def test_extracts_openai_compatible_response(self) -> None: + response = LlamaCppService._extract_response( + {"choices": [{"message": {"content": "hello"}}]} + ) + + self.assertEqual(response, "hello") + + def test_status_reports_unreachable_server(self) -> None: + def raise_request_error(*_args, **_kwargs): + raise requests.ConnectionError("offline") + + status = LlamaCppService.status( + which_func=lambda _name: "llama-server", + get_func=raise_request_error, + ) + + self.assertFalse(status.available) + self.assertIn("not reachable", status.detail) + + def test_status_uses_configured_server_path(self) -> None: + def ok_response(*_args, **_kwargs): + response = requests.Response() + response.status_code = 200 + return response + + status = LlamaCppService.status( + which_func=lambda _name: None, + get_func=ok_response, + server_path=__file__, + ) + + self.assertTrue(status.available) + + def test_local_file_status(self) -> None: + self.assertEqual(local_file_status(""), "not configured") + self.assertEqual(local_file_status("missing-file.gguf"), "missing") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_local_backend_config.py b/tests/unit/test_local_backend_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d7ef3397f916dfa035028c2baddf4c60e9627f14 --- /dev/null +++ b/tests/unit/test_local_backend_config.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from models.local_backend_config import ( + LocalBackendConfig, + build_llama_server_command, + load_local_backend_config, + local_backend_summary, + save_local_backend_config, +) + + +class LocalBackendConfigTest(unittest.TestCase): + def test_loads_defaults_when_config_is_missing(self) -> None: + config = load_local_backend_config("missing-local-backends.yaml") + + self.assertEqual(config.llama_cpp_server_url, "http://127.0.0.1:8080") + self.assertEqual(config.llama_server_path, "") + self.assertEqual(config.openai_compatible_base_url, "http://127.0.0.1:1234") + self.assertEqual(config.openai_compatible_model_name, "") + self.assertEqual(config.gguf_path, "") + + def test_saves_and_loads_local_backend_config(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "local_backends.yaml" + expected = LocalBackendConfig( + llama_cpp_server_url="http://127.0.0.1:9090", + llama_server_path="C:/llama/llama-server.exe", + openai_compatible_base_url="http://local.test:1234", + openai_compatible_model_name="loaded-model", + gguf_path="model.gguf", + mmproj_path="mmproj.gguf", + n_ctx=8192, + n_gpu_layers=12, + ) + + saved = save_local_backend_config(expected, path) + actual = load_local_backend_config(saved) + + self.assertEqual(actual, expected) + + def test_builds_llama_server_command_only_when_model_is_configured(self) -> None: + self.assertEqual(build_llama_server_command(LocalBackendConfig()), []) + + command = build_llama_server_command( + LocalBackendConfig( + llama_server_path="C:/llama/llama-server.exe", + gguf_path="model.gguf", + mmproj_path="mmproj.gguf", + ) + ) + + self.assertEqual( + command, + [ + "C:/llama/llama-server.exe", + "-m", + "model.gguf", + "--mmproj", + "mmproj.gguf", + ], + ) + + def test_summary_records_no_startup_download_or_auto_load(self) -> None: + summary = local_backend_summary(LocalBackendConfig(gguf_path="model.gguf")) + + self.assertFalse(summary["startup_downloads"]) + self.assertFalse(summary["auto_model_load"]) + self.assertIn("llama-server -m model.gguf", summary["llama_server_command"]) + self.assertEqual(summary["openai_compatible_base_url"], "http://127.0.0.1:1234") + self.assertEqual(summary["files"][0][0], "llama-server") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lora_trainer.py b/tests/unit/test_lora_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..dd419f2baac7c0591acb5f360348b467dfed1bc7 --- /dev/null +++ b/tests/unit/test_lora_trainer.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from training.lora_trainer import ( + build_lora_training_request, + lora_dependency_report, + vision_finetuning_plan, +) + + +class LoraTrainerTest(unittest.TestCase): + def test_dependency_report_is_serializable(self) -> None: + report = lora_dependency_report().as_dict() + + self.assertIn("ready", report) + self.assertIn("peft_available", report) + self.assertIn("trl_available", report) + + def test_builds_non_executing_lora_request(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + dataset = Path(tmp) / "train.jsonl" + dataset.write_text('{"prompt":"hello","correction":"world"}\n', encoding="utf-8") + + request = build_lora_training_request( + "minicpm5_1b", + str(dataset), + rank=8, + epochs=2, + output_root=Path(tmp) / "out", + ) + + data = request.as_dict() + self.assertFalse(data["execute_training"]) + self.assertEqual(data["plan"]["lora"]["rank"], 8) + self.assertIn("--model-id", data["command_preview"]) + + def test_vision_finetuning_plan_mentions_swift_and_llama_factory(self) -> None: + plan = vision_finetuning_plan() + + self.assertFalse(plan["implemented"]) + self.assertIn("SWIFT", plan["recommended_tools"]) + self.assertIn("LLaMA-Factory", plan["recommended_tools"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_mcp_bridge.py b/tests/unit/test_mcp_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..ac435425d393409271001750b344bba0dee87a57 --- /dev/null +++ b/tests/unit/test_mcp_bridge.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import unittest + +from mcp_tools.bridge import MCP_PATH, invoke_mcp_tool, mcp_manifest, mcp_tool_definitions + + +class McpBridgeTest(unittest.TestCase): + def test_manifest_documents_gradio_native_path(self) -> None: + manifest = mcp_manifest() + + self.assertEqual(manifest["path"], MCP_PATH) + self.assertEqual(manifest["mode"], "gradio_native_mcp_server") + self.assertIn("launch(mcp_server=True)", manifest["served_by"]) + self.assertTrue(manifest["tools"]) + + def test_tool_definitions_include_registry_tools(self) -> None: + names = [definition.name for definition in mcp_tool_definitions()] + + self.assertIn("safe_calculator", names) + self.assertIn("model_inference", names) + + def test_invokes_registered_tool(self) -> None: + result = invoke_mcp_tool("safe_calculator", {"expression": "2 + 2"}) + + self.assertEqual(result.name, "safe_calculator") + self.assertEqual(result.payload["value"], 4) + + def test_rejects_unknown_tool(self) -> None: + with self.assertRaises(KeyError): + invoke_mcp_tool("missing", {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_mcp_tools.py b/tests/unit/test_mcp_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..03960031f7c7b7ea78afc45a6881bafbb19d5275 --- /dev/null +++ b/tests/unit/test_mcp_tools.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from mcp_tools.tools import ( + dataset_stats_tool, + hf_dataset_preview_tool, + model_inference_tool, + safe_calculator_tool, + tool_registry, + vindex_plan_tool, +) + + +class McpToolsTest(unittest.TestCase): + def test_dataset_stats_tool(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "data.csv" + path.write_text("prompt,response\nhello,world\nempty,\n", encoding="utf-8") + + result = dataset_stats_tool(str(path)) + + self.assertEqual(result.name, "dataset_stats") + self.assertEqual(result.payload["rows"], 2) + self.assertEqual(result.payload["non_empty_by_column"]["response"], 1) + + def test_safe_calculator_tool(self) -> None: + result = safe_calculator_tool("2 + 3 * 4") + + self.assertEqual(result.payload["value"], 14) + + def test_safe_calculator_rejects_names(self) -> None: + with self.assertRaises(ValueError): + safe_calculator_tool("__import__('os').system('echo nope')") + + def test_model_inference_tool_uses_placeholder_backend(self) -> None: + result = model_inference_tool("Hello") + + self.assertIn("Placeholder response", result.payload["response"]) + + def test_hf_dataset_preview_tool_reports_missing_optional_package(self) -> None: + result = hf_dataset_preview_tool("demo/dataset") + + self.assertEqual(result.name, "hf_dataset_preview") + self.assertIn("status", result.payload) + + def test_tool_registry_contains_expected_tools(self) -> None: + registry = tool_registry() + + self.assertIn("dataset_stats", registry) + self.assertIn("safe_calculator", registry) + self.assertIn("model_inference", registry) + self.assertIn("vindex_plan", registry) + + def test_vindex_plan_tool_returns_non_executing_plan(self) -> None: + result = vindex_plan_tool("logit_lens", {"model_id": "demo", "text": "hello"}) + + self.assertEqual(result.name, "vindex_plan") + self.assertFalse(result.payload["ready_for_execution"]) + self.assertEqual(result.payload["call_plan"]["method"], "logit_lens") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_minicpm_vision.py b/tests/unit/test_minicpm_vision.py new file mode 100644 index 0000000000000000000000000000000000000000..04442d0da3fbc66547c89aabebb3862b0a13df9d --- /dev/null +++ b/tests/unit/test_minicpm_vision.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import unittest +from typing import cast + +from models.minicpm_vision import MiniCPMVisionConfig, MiniCPMVisionService +from models.model_catalog import load_model_catalog + + +class MiniCPMVisionServiceTest(unittest.TestCase): + def test_status_reports_transformers_availability(self) -> None: + status = MiniCPMVisionService.status() + + self.assertEqual(status.name, "transformers-vision") + self.assertIsInstance(status.available, bool) + + def test_unavailable_or_missing_image_path_does_not_load_weights(self) -> None: + catalog = load_model_catalog("config/models.yaml") + service = MiniCPMVisionService(catalog["minicpm_v46"]) + + response = service.vision_chat(False, "Describe it.") + + self.assertIn("[MiniCPM vision unavailable]", response) + + def test_generation_kwargs_are_explicit(self) -> None: + catalog = load_model_catalog("config/models.yaml") + service = MiniCPMVisionService( + catalog["minicpm_v46"], + MiniCPMVisionConfig(max_new_tokens=64, temperature=0.1, do_sample=False), + ) + + self.assertEqual( + service.generation_kwargs(), + {"max_new_tokens": 64, "temperature": 0.1, "do_sample": False}, + ) + + def test_formats_image_text_prompt_with_thinking_instruction(self) -> None: + messages = MiniCPMVisionService.format_messages( + "Read the label.", + image="image-object", + thinking=True, + ) + + self.assertEqual(messages[0]["role"], "user") + self.assertEqual(messages[0]["content"][0]["type"], "image") + self.assertEqual(messages[0]["content"][1]["type"], "text") + self.assertIn("Think carefully", messages[0]["content"][1]["text"]) + + def test_video_support_plan_is_documented_but_not_enabled(self) -> None: + plan = MiniCPMVisionService.video_support_plan() + next_steps = cast(list[str], plan["next_steps"]) + + self.assertFalse(plan["implemented"]) + self.assertTrue(any("Sample frames locally" in step for step in next_steps)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_model_catalog.py b/tests/unit/test_model_catalog.py new file mode 100644 index 0000000000000000000000000000000000000000..df7dc9e18e0324621e43587c7020a7ff4d6520ca --- /dev/null +++ b/tests/unit/test_model_catalog.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import unittest + +from models.model_catalog import load_model_catalog, model_choices, model_summary, validate_catalog + + +class ModelCatalogTest(unittest.TestCase): + def test_loads_prd_models_from_config(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + self.assertIn("minicpm5_1b", catalog) + self.assertIn("minicpm41_8b", catalog) + self.assertIn("minicpm_v46", catalog) + self.assertIn("minicpm_v46_thinking", catalog) + + def test_model_choices_filter_by_type(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + text_models = model_choices(catalog, "text") + vision_models = model_choices(catalog, "vision") + + self.assertIn("minicpm5_1b", text_models) + self.assertIn("minicpm_v46", vision_models) + self.assertNotIn("minicpm_v46", text_models) + + def test_summary_contains_hackathon_compliance_fields(self) -> None: + catalog = load_model_catalog("config/models.yaml") + summary = model_summary(catalog["minicpm5_1b"]) + + self.assertEqual(summary["hf_id"], "openbmb/MiniCPM5-1B") + self.assertLessEqual(summary["parameters_b"], 32) + self.assertIn("gguf", summary) + self.assertIn("backend_capabilities", summary) + self.assertIn("llama.cpp", summary["backend_capabilities"]) + self.assertIn("trust_remote_code", summary) + + def test_all_models_have_backend_capability_metadata(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + for model in catalog.values(): + self.assertIn(model.backend, model.backend_capabilities) + self.assertTrue(model.backend_capabilities[model.backend]) + + def test_catalog_defaults_to_real_transformers_backends(self) -> None: + catalog = load_model_catalog("config/models.yaml") + warnings = validate_catalog(catalog) + + self.assertFalse(any("placeholder backend" in warning for warning in warnings)) + self.assertEqual(catalog["minicpm5_1b"].backend, "transformers") + self.assertEqual(catalog["minicpm_v46"].backend, "transformers") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_ocr_pipeline.py b/tests/unit/test_ocr_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..642fda7e9b7c81e888145662bbfa4cf0ea4870ca --- /dev/null +++ b/tests/unit/test_ocr_pipeline.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from datasets.field_notes import FieldNote, FieldNoteStore +from datasets.ocr import ( + export_corrected_ocr_notes, + import_uncertain_predictions, + load_ocr_predictions, + ocr_import_summary, + uncertain_predictions, +) +from ui.notes_tab import import_ocr_predictions, preview_ocr_predictions + + +class OCRPipelineTest(unittest.TestCase): + def test_loads_ocr_predictions_from_jsonl(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ocr.jsonl" + path.write_text( + json.dumps( + { + "source_path": "receipt.png", + "text": "Total 12.30", + "confidence": 0.91, + "page": 1, + } + ) + + "\n", + encoding="utf-8", + ) + + predictions = load_ocr_predictions(path) + + self.assertEqual(len(predictions), 1) + self.assertEqual(predictions[0].source_path, "receipt.png") + self.assertEqual(predictions[0].confidence, 0.91) + self.assertEqual(predictions[0].page, "1") + + def test_loads_ocr_predictions_from_csv_alias_columns(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ocr.csv" + path.write_text( + "image_path,prediction,score\nlabel.png,Best before 2026,0.72\n", + encoding="utf-8", + ) + + predictions = load_ocr_predictions(path) + + self.assertEqual(predictions[0].source_path, "label.png") + self.assertEqual(predictions[0].text, "Best before 2026") + self.assertEqual(predictions[0].confidence, 0.72) + + def test_filters_uncertain_predictions_by_threshold_and_empty_text(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ocr.jsonl" + rows = [ + {"source_path": "a.png", "text": "clear", "confidence": 0.95}, + {"source_path": "b.png", "text": "maybe", "confidence": 0.7}, + {"source_path": "c.png", "text": "", "confidence": 0.99}, + ] + path.write_text( + "\n".join(json.dumps(row) for row in rows), + encoding="utf-8", + ) + + uncertain = uncertain_predictions(load_ocr_predictions(path), 0.8) + + self.assertEqual([item.source_path for item in uncertain], ["b.png", "c.png"]) + + def test_imports_uncertain_predictions_to_field_notes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + predictions_path = Path(tmp) / "ocr.jsonl" + predictions_path.write_text( + "\n".join( + [ + json.dumps( + { + "source_path": "uncertain.png", + "text": "hel1o", + "confidence": 0.5, + } + ), + json.dumps( + { + "source_path": "confident.png", + "text": "hello", + "confidence": 0.99, + } + ), + ] + ), + encoding="utf-8", + ) + store = FieldNoteStore(Path(tmp) / "field_notes.csv") + + imported = import_uncertain_predictions( + store, + load_ocr_predictions(predictions_path), + "minicpm5_1b", + confidence_threshold=0.8, + ) + + notes = store.list_notes(tag="ocr") + self.assertEqual(imported, 1) + self.assertEqual(len(notes), 1) + self.assertEqual(notes[0].response, "hel1o") + self.assertEqual(notes[0].image_path, "uncertain.png") + self.assertFalse(notes[0].use_for_training) + + def test_exports_corrected_ocr_notes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + store = FieldNoteStore(Path(tmp) / "field_notes.csv") + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Review OCR text for receipt.png.", + response="Tota1", + correction="Total", + tags="ocr,uncertain", + image_path="receipt.png", + ) + ) + store.save( + FieldNote.create( + model_id="minicpm5_1b", + prompt="Other correction", + response="abc", + correction="abc", + tags="demo", + ) + ) + + output = export_corrected_ocr_notes(store, Path(tmp) / "ocr_corrections.jsonl") + + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["source_path"], "receipt.png") + self.assertEqual(rows[0]["predicted_text"], "Tota1") + self.assertEqual(rows[0]["corrected_text"], "Total") + + def test_ocr_import_summary_reports_uncertain_sample(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ocr.jsonl" + path.write_text( + json.dumps({"source_path": "a.png", "text": "?", "confidence": 0.1}), + encoding="utf-8", + ) + + summary = ocr_import_summary(path, 0.8) + + self.assertEqual(summary["rows"], 1) + self.assertEqual(summary["uncertain_rows"], 1) + self.assertEqual(summary["confidence_threshold"], 0.8) + + def test_notes_tab_ocr_preview_callback_reports_missing_path(self) -> None: + self.assertEqual( + preview_ocr_predictions("", 0.8), + {"error": "Enter a local OCR prediction file path."}, + ) + + def test_notes_tab_ocr_import_callback_binds_store(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ocr.jsonl" + path.write_text( + json.dumps({"source_path": "a.png", "text": "?", "confidence": 0.1}), + encoding="utf-8", + ) + store = FieldNoteStore(Path(tmp) / "field_notes.csv") + + status = import_ocr_predictions(store, "minicpm5_1b", str(path), 0.8) + + self.assertIn("Imported 1 uncertain OCR predictions", status) + self.assertEqual(len(store.list_notes(tag="ocr")), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_ollama_service.py b/tests/unit/test_ollama_service.py new file mode 100644 index 0000000000000000000000000000000000000000..83397d8925b808e885f29a232672653d9d8bc9b7 --- /dev/null +++ b/tests/unit/test_ollama_service.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import unittest + +import requests + +import models.ollama_service as ollama_module +from models.model_catalog import load_model_catalog +from models.ollama_service import OllamaService + + +class OllamaServiceTest(unittest.TestCase): + def test_status_reports_missing_executable(self) -> None: + status = OllamaService.status(which_func=lambda _name: None) + + self.assertFalse(status.available) + self.assertIn("not found", status.detail) + + def test_chat_returns_clear_unavailable_message(self) -> None: + catalog = load_model_catalog("config/models.yaml") + + class UnavailableOllamaService(OllamaService): + @staticmethod + def status(*_args, **_kwargs) -> ollama_module.BackendStatus: + return ollama_module.BackendStatus("ollama", False, "Ollama test unavailable.") + + response = UnavailableOllamaService(catalog["minicpm5_1b"]).chat("", "Hello") + + self.assertIn("[Ollama unavailable]", response) + self.assertIn("Ollama test unavailable.", response) + + def test_lists_local_models(self) -> None: + response = requests.Response() + response.status_code = 200 + response._content = ( # pylint: disable=protected-access + b'{"models": [{"name": "minicpm:latest"}, {"name": "llama:latest"}]}' + ) + + models = OllamaService.list_local_models(get_func=lambda *_args, **_kwargs: response) + + self.assertEqual(models, ["minicpm:latest", "llama:latest"]) + + def test_list_local_models_returns_empty_when_unreachable(self) -> None: + def raise_error(*_args, **_kwargs): + raise requests.ConnectionError("offline") + + self.assertEqual(OllamaService.list_local_models(get_func=raise_error), []) + + def test_builds_explicit_pull_command(self) -> None: + self.assertEqual( + OllamaService.pull_command(" minicpm:latest "), + ["ollama", "pull", "minicpm:latest"], + ) + self.assertEqual(OllamaService.pull_command(""), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_openai_compatible_service.py b/tests/unit/test_openai_compatible_service.py new file mode 100644 index 0000000000000000000000000000000000000000..50732fe5d1c681c5d8e3acc55638ab72177af0ae --- /dev/null +++ b/tests/unit/test_openai_compatible_service.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import json +import unittest +from typing import Any + +import requests + +from models.model_catalog import load_model_catalog +from models.openai_compatible_service import ( + OpenAICompatibleConfig, + OpenAICompatibleService, +) + + +def _json_response(payload: dict[str, Any], status_code: int = 200) -> requests.Response: + response = requests.Response() + response.status_code = status_code + response._content = json.dumps(payload).encode("utf-8") + return response + + +class CapturingPost: + def __init__(self) -> None: + self.payload: dict[str, Any] = {} + self.url = "" + + def __call__(self, url: str, **kwargs: Any) -> requests.Response: + self.url = url + self.payload = dict(kwargs["json"]) + return _json_response({"choices": [{"message": {"content": "local answer"}}]}) + + +class OpenAICompatibleServiceTest(unittest.TestCase): + def setUp(self) -> None: + self.model = load_model_catalog("config/models.yaml")["minicpm5_1b"] + + def test_status_reports_reachable_server(self) -> None: + def get_models(url: str, **kwargs: Any) -> requests.Response: + self.assertEqual(url, "http://local.test/v1/models") + self.assertEqual(kwargs["timeout"], 2) + return _json_response({"data": [{"id": "loaded-model"}]}) + + status = OpenAICompatibleService.status("http://local.test", get_models) + + self.assertTrue(status.available) + self.assertEqual(status.name, "openai-compatible") + + def test_status_reports_unreachable_server(self) -> None: + def get_models(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + raise requests.ConnectionError("not listening") + + status = OpenAICompatibleService.status("http://local.test", get_models) + + self.assertFalse(status.available) + self.assertIn("not reachable", status.detail) + + def test_chat_posts_openai_compatible_payload_with_model_override(self) -> None: + def get_models(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + return _json_response({"data": [{"id": "lm-studio-model"}]}) + + post_chat = CapturingPost() + service = OpenAICompatibleService( + self.model, + OpenAICompatibleConfig( + base_url="http://local.test/", + model_name="lm-studio-model", + api_key="local-key", + ), + get_func=get_models, + post_func=post_chat, + ) + + answer = service.chat("be concise", "hello") + + self.assertEqual(answer, "local answer") + self.assertEqual(post_chat.url, "http://local.test/v1/chat/completions") + self.assertEqual(post_chat.payload["model"], "lm-studio-model") + self.assertEqual(post_chat.payload["messages"][0]["role"], "system") + self.assertEqual(post_chat.payload["messages"][1]["content"], "hello") + + def test_chat_returns_unavailable_message_without_posting(self) -> None: + def get_models(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + raise requests.ConnectionError("server down") + + def post_chat(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + raise AssertionError("chat request should not be sent when status fails") + + service = OpenAICompatibleService( + self.model, + OpenAICompatibleConfig(base_url="http://local.test"), + get_func=get_models, + post_func=post_chat, + ) + + answer = service.chat("system", "prompt") + + self.assertIn("[OpenAI-compatible backend unavailable]", answer) + self.assertIn("server down", answer) + + def test_chat_returns_request_failure_message(self) -> None: + def get_models(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + return _json_response({"data": [{"id": "loaded-model"}]}) + + def post_chat(url: str, **kwargs: Any) -> requests.Response: + del url, kwargs + raise requests.Timeout("slow local model") + + service = OpenAICompatibleService( + self.model, + OpenAICompatibleConfig(base_url="http://local.test"), + get_func=get_models, + post_func=post_chat, + ) + + answer = service.chat("system", "prompt") + + self.assertIn("[OpenAI-compatible request failed]", answer) + self.assertIn("slow local model", answer) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_plant_reference_app.py b/tests/unit/test_plant_reference_app.py new file mode 100644 index 0000000000000000000000000000000000000000..cf17e38660f31ab2bfd49ec9892847c855ddf9c4 --- /dev/null +++ b/tests/unit/test_plant_reference_app.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from core.deployment import DeploymentPolicy +from datasets.field_notes import FieldNote, FieldNoteStore +from plant.app import build_app, load_config +from plant.plant_loader import ( + FieldNotesPlantExporter, + LocalFolderLoader, + SpeciesIndexBuilder, + demo_species, +) +from plant.plant_service import DemoPlantVisionService, extract_json_object, parse_plant_response +from plant.plant_tab import ( + identify_plant_callback, + plant_training_plan, + render_result_card, + save_field_note_callback, + species_table, +) +from plant.plant_tools import dataset_stats, search_species, set_services, training_plan +from plant.training import build_plant_training_plan, write_llamafactory_dataset_info + + +class PlantReferenceAppTest(unittest.TestCase): + def test_plant_config_loads_domain_and_model(self) -> None: + config = load_config("plant/models.yaml") + + self.assertEqual(config["domain"]["name"], "plant_discovery") + self.assertEqual(config["models"]["plant_vlm"]["hf_id"], "openbmb/MiniCPM-V-4.6") + + def test_no_model_app_builds_gradio_blocks(self) -> None: + demo = build_app(no_model=True) + + self.assertEqual(type(demo).__name__, "Blocks") + + def test_space_policy_rejects_no_model_mode(self) -> None: + import plant.app as plant_app + + original = plant_app.current_policy + plant_app.current_policy = lambda: DeploymentPolicy("space") # type: ignore[assignment] + try: + with self.assertRaises(ValueError): + build_app(no_model=True) + finally: + plant_app.current_policy = original # type: ignore[assignment] + + def test_default_app_builds_openbmb_service_without_loading_weights(self) -> None: + demo = build_app() + + self.assertEqual(type(demo).__name__, "Blocks") + + def test_demo_service_returns_structured_plant_result(self) -> None: + result = DemoPlantVisionService().identify(object(), force_thinking=True) + + self.assertEqual(result.latin_name, "Bellis perennis") + self.assertGreater(result.confidence, 0.8) + self.assertEqual(result.to_dict()["family"], "Asteraceae") + + def test_demo_service_reports_no_llm_usage(self) -> None: + status = DemoPlantVisionService().service_status() + + self.assertFalse(status["uses_llm"]) + self.assertEqual(status["mode"], "demo") + + def test_extract_json_repairs_common_model_wrapping(self) -> None: + parsed = extract_json_object('```json\n{"latin_name":"Rosa canina",}\n```') + + self.assertEqual(parsed["latin_name"], "Rosa canina") + + def test_parse_plant_response_builds_schema(self) -> None: + result = parse_plant_response( + '{"common_name":"Oak","latin_name":"Quercus robur","confidence":0.91}', + model_used="demo", + ) + + self.assertEqual(result.genus, "Quercus") + self.assertEqual(result.model_used, "demo") + + def test_local_folder_loader_uses_species_folder_names(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + species_dir = Path(tmp) / "Acer_palmatum" + species_dir.mkdir() + (species_dir / "leaf.jpg").write_bytes(b"not a real image but good metadata") + + loader = LocalFolderLoader(tmp) + + self.assertEqual(loader.species_list(), ["Acer palmatum"]) + self.assertEqual(loader.iter_records()[0].latin_name, "Acer palmatum") + + def test_species_index_falls_back_to_demo_species(self) -> None: + index = SpeciesIndexBuilder(root="missing-plant-root").build({"datasets": {}}) + + self.assertIn("Bellis perennis", index) + + def test_species_table_filters_by_query_and_family(self) -> None: + rows = species_table(demo_species(), "rose", "Rosaceae") + + self.assertEqual(rows[0][0], "Rosa canina") + + def test_render_result_card_escapes_model_text(self) -> None: + html = render_result_card( + { + "common_name": "", + "latin_name": "Bellis perennis", + "confidence": 0.5, + } + ) + + self.assertIn("<script>", html) + self.assertNotIn("