diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..be86c8426eedf3224e93d8677e53de92c95abb76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +potato/static/vendor/font-awesome-6.7.2/webfonts/fa-brands-400.ttf filter=lfs diff=lfs merge=lfs -text +potato/static/vendor/font-awesome-6.7.2/webfonts/fa-brands-400.woff2 filter=lfs diff=lfs merge=lfs -text +potato/static/vendor/font-awesome-6.7.2/webfonts/fa-solid-900.ttf filter=lfs diff=lfs merge=lfs -text +potato/static/vendor/font-awesome-6.7.2/webfonts/fa-solid-900.woff2 filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c33bd688429da992c0e840006a809d8a17f34035 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM python:3.11-slim + +# Create non-root user (HF Spaces requires UID 1000) +RUN useradd -m -u 1000 potato + +# Install system dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy requirements first for layer caching +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt gunicorn + +# Copy the application source +COPY . . + +# Install potato +RUN pip install --no-cache-dir -e . + +# Copy entrypoint +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Create directories for output +RUN mkdir -p /app/annotation_output && \ + chown -R potato:potato /app + +# Switch to non-root user +USER potato + +# HuggingFace Spaces expects port 7860 +EXPOSE 7860 + +ENV POTATO_CONFIG=config.yaml +ENV PORT=7860 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index e8533588bac37843bf9f56f02c844dd0b4e2f879..9fe77e2a06890b88abcd758f03529f5ceaff1e3a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,47 @@ --- -title: Openai Evaluation -emoji: ๐Ÿ  -colorFrom: red -colorTo: pink +title: Potato โ€” OpenAI Trace Evaluation +emoji: ๐Ÿฅ” +colorFrom: indigo +colorTo: purple sdk: docker +app_port: 7860 pinned: false +license: apache-2.0 +tags: + - annotation + - potato + - agent + - agent-traces + - openai --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Potato โ€” OpenAI Trace Evaluation + +Annotate OpenAI agent / tool-use traces. + +A live demo of **[Potato](https://www.potatoannotator.com)** โ€” the free, self-hosted annotation platform for NLP, +agentic, and GenAI research, configured entirely through YAML. +Visit **[www.potatoannotator.com](https://www.potatoannotator.com)** for docs, the schema gallery, and more demos. + +## Try it out + +1. Enter any username to log in (no password required). +2. Read the item shown in the main panel. +3. Annotate using the schemes on the right. +4. Click **Next** to continue. + +> **Run your own copy:** click the **โ‹ฎ โ†’ Duplicate this Space** button (top-right) to launch +> this exact demo in your own account on free hardware โ€” change the data and config to make it yours. + +> Annotations in this demo are ephemeral. To collect and keep data, deploy your own +> Space โ€” see the [deployment guide](https://github.com/davidjurgens/potato/blob/master/deployment/huggingface-spaces/deploy.md). + +## About Potato + +Potato supports 20+ annotation types โ€” text, spans, images, audio, video, documents, +and agent traces โ€” with AI-assisted labeling, quality control, and adjudication. + +๐Ÿฅ” **Website: [www.potatoannotator.com](https://www.potatoannotator.com)**  ยท  +[Documentation](https://www.potatoannotator.com)  ยท  +[GitHub](https://github.com/davidjurgens/potato)  ยท  +[All demos](https://github.com/davidjurgens/potato/blob/master/docs/data-export/potato_on_huggingface.md) diff --git a/annotation_output/.gitkeep b/annotation_output/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f06e9ae2e56f3383225b9b268f6a4be5fe76d614 --- /dev/null +++ b/config.yaml @@ -0,0 +1,53 @@ +annotation_task_name: OpenAI Trace Evaluation +data_files: +- data/openai-traces.json +item_properties: + id_key: id + text_key: task_description +task_dir: . +output_annotation_dir: annotation_output +output_annotation_format: json +instance_display: + layout: + direction: vertical + gap: 16px + fields: + - key: task_description + type: text + label: Task + - key: metadata_table + type: spreadsheet + label: Trace Info + display_options: + compact: true + max_height: 120 + - key: conversation + type: dialogue + label: Agent Trace + display_options: + show_turn_numbers: true + alternating_shading: true +annotation_schemes: +- annotation_type: radio + name: task_success + description: Did the agent complete the task? + labels: + - name: success + tooltip: Fully completed correctly + - name: partial + tooltip: Partially completed or with errors + - name: failure + tooltip: Not completed + sequential_key_binding: true +- annotation_type: likert + name: response_quality + description: Rate the quality of the agent's response + min_label: Poor + max_label: Excellent + size: 5 +- annotation_type: text + name: notes + description: Additional notes +user_config: + allow_all_users: true +require_no_password: true diff --git a/data/openai-traces.json b/data/openai-traces.json new file mode 100644 index 0000000000000000000000000000000000000000..908984beaa306d33e7ba9c0565519e9f9780054e --- /dev/null +++ b/data/openai-traces.json @@ -0,0 +1,39 @@ +[ + { + "id": "chatcmpl-001", + "model": "gpt-4", + "messages": [ + {"role": "system", "content": "You are a helpful travel assistant."}, + {"role": "user", "content": "Find me the cheapest flight from JFK to London next Friday."}, + {"role": "assistant", "content": "I'll search for flights from JFK to London for next Friday.", "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "search_flights", "arguments": "{\"origin\": \"JFK\", \"destination\": \"LHR\", \"date\": \"2024-03-15\"}"}} + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": "Found 5 flights:\n1. BA117 - $450 (direct, 7h)\n2. VS3 - $420 (direct, 7h15m)\n3. AA100 - $480 (direct, 7h30m)\n4. UA110 - $510 (1 stop, 10h)\n5. DL400 - $395 (1 stop, 11h)"}, + {"role": "assistant", "content": "I found several options. The cheapest is DL400 at $395 with one stop (11 hours), but VS3 at $420 is the best value for a direct flight (7h15m). Would you like me to book either of these?"} + ], + "usage": {"prompt_tokens": 245, "completion_tokens": 89, "total_tokens": 334} + }, + { + "id": "chatcmpl-002", + "model": "gpt-4", + "messages": [ + {"role": "system", "content": "You are a code review assistant."}, + {"role": "user", "content": "Review this Python function for bugs:\ndef divide(a, b):\n return a / b"}, + {"role": "assistant", "content": "I see a potential issue. The function doesn't handle division by zero. Here's a fix:\n\n```python\ndef divide(a, b):\n if b == 0:\n raise ValueError(\"Cannot divide by zero\")\n return a / b\n```\n\nI'd also recommend adding type hints and a docstring."} + ], + "usage": {"prompt_tokens": 120, "completion_tokens": 65, "total_tokens": 185} + }, + { + "id": "chatcmpl-003", + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "What's the capital of France and its population?"}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_2", "type": "function", "function": {"name": "lookup_city", "arguments": "{\"city\": \"Paris\", \"fields\": [\"population\", \"country\"]}"}} + ]}, + {"role": "tool", "tool_call_id": "call_2", "content": "{\"city\": \"Paris\", \"country\": \"France\", \"population\": 2161000, \"metro_population\": 12213364}"}, + {"role": "assistant", "content": "The capital of France is Paris, with a city population of about 2.16 million and a metropolitan area population of approximately 12.2 million."} + ], + "usage": {"prompt_tokens": 95, "completion_tokens": 42, "total_tokens": 137} + } +] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..140f2f12b5e8e1f5a7c8a10f92572611411743f0 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +# Configuration +CONFIG_FILE="${POTATO_CONFIG:-config.yaml}" +PORT="${PORT:-7860}" +WORKERS="${GUNICORN_WORKERS:-2}" +THREADS="${GUNICORN_THREADS:-4}" +TIMEOUT="${GUNICORN_TIMEOUT:-120}" + +echo "Starting Potato Demo Space..." +echo " Config: ${CONFIG_FILE}" +echo " Port: ${PORT}" +echo " Workers: ${WORKERS}" + +# Validate config exists +if [ ! -f "${CONFIG_FILE}" ]; then + echo "ERROR: Config file not found: ${CONFIG_FILE}" + exit 1 +fi + +# Start with gunicorn using the factory pattern +exec gunicorn \ + --bind "0.0.0.0:${PORT}" \ + --workers "${WORKERS}" \ + --threads "${THREADS}" \ + --timeout "${TIMEOUT}" \ + --access-logfile - \ + --error-logfile - \ + "potato.flask_server:create_app('${CONFIG_FILE}')" diff --git a/layouts/task_layout.html b/layouts/task_layout.html new file mode 100644 index 0000000000000000000000000000000000000000..3b6bc21790d64e1e7200b633627b8c39f183539c --- /dev/null +++ b/layouts/task_layout.html @@ -0,0 +1,173 @@ + + + + + + +
+ +
+ +
+ Did the agent complete the task? +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+ Rate the quality of the agent's response +
+
Poor
+
+
+ +
+ + + 1 +
+ +
+ + + 2 +
+ +
+ + + 3 +
+ +
+ + + 4 +
+ +
+ + + 5 +
+ +
+
Excellent
+
+ +
+ + +
+ +
+ + Additional notes + +
+ + + +
+ +
diff --git a/potato/__init__.py b/potato/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a8e89cfdaffad4d00849cff05a16624488cd83b1 --- /dev/null +++ b/potato/__init__.py @@ -0,0 +1,45 @@ +""" +Potato Annotation Platform + +A flexible, web-based platform for text annotation tasks. + +This package provides a comprehensive annotation system with the following features: +- Multi-phase annotation workflows (consent, instructions, training, annotation, post-study) +- Support for various annotation types (labels, spans, text, likert scales, best-worst scaling) +- User authentication and session management +- Active learning capabilities +- Admin dashboard for monitoring progress +- Configurable assignment strategies +- Multi-language and multi-task support + +Main Components: +- flask_server: Core Flask application and server logic +- routes: HTTP route handlers and request processing +- user_state_management: User progress tracking and state persistence +- item_state_management: Data item management and assignment +- authentificaton: User authentication backends +- admin: Admin dashboard functionality +- activelearning: Active learning algorithms and model training + +Usage: + from potato.flask_server import create_app + app = create_app() + app.run() +""" + +from .flask_server import create_app + +__version__ = "2.6.0" +__author__ = "Potato Annotation Platform Team" +__description__ = "A flexible, web-based platform for text annotation tasks" + + +def __getattr__(name): + """Lazy imports for optional heavy dependencies.""" + if name == "load_as_dataset": + from .datasets_integration import load_as_dataset + return load_as_dataset + if name == "load_annotations": + from .datasets_integration import load_annotations + return load_annotations + raise AttributeError(f"module 'potato' has no attribute {name!r}") \ No newline at end of file diff --git a/potato/__main__.py b/potato/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..a023e5eb55074b783311863e90bd6b9ae226598c --- /dev/null +++ b/potato/__main__.py @@ -0,0 +1,3 @@ +from potato.flask_server import main + +main() diff --git a/potato/active_learning_manager.py b/potato/active_learning_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..023024201b664b17f7bb3e603bc3d7ed00a25fc8 --- /dev/null +++ b/potato/active_learning_manager.py @@ -0,0 +1,1623 @@ +""" +Enhanced Active Learning Manager with Database Persistence + +This module provides a comprehensive active learning system with optional +database persistence, model saving, LLM integration, and multiple query +strategies including uncertainty sampling, diversity sampling, BADGE, BALD, +and hybrid combinations. + +References: + [1] Ash et al. (2020) "Deep Batch Active Learning by Diverse, Uncertain + Gradient Lower Bounds" (BADGE). ICLR 2020. + [2] Houlsby et al. (2011) "Bayesian Active Learning for Classification + and Preference Learning" (BALD). + [3] Bayer et al. (2024) "ActiveLLM: Large Language Model-Based Active + Learning for Textual Few-Shot Scenarios". TACL. + [4] Yuan et al. (2024) "Hide and Seek in Noise Labels: Noise-Robust + Collaborative Active Learning" (NoiseAL). ACL 2024. + [5] Mavromatis et al. (2024) "CoverICL: Selective Annotation for + In-Context Learning via Active Graph Coverage". EMNLP 2024. +""" + +import threading +import logging +import time +import os +import pickle +import json +from typing import Dict, List, Optional, Tuple, Any, Union +from collections import defaultdict, Counter +import dataclasses +from dataclasses import dataclass, field, asdict +from enum import Enum +import random +import queue +from datetime import datetime +from abc import ABC, abstractmethod + +from sklearn.pipeline import Pipeline +from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import RandomForestClassifier +from sklearn.svm import SVC +from sklearn.metrics import accuracy_score, classification_report +import numpy as np + +from potato.item_state_management import ItemStateManager, get_item_state_manager +from potato.user_state_management import get_user_state_manager + + +logger = logging.getLogger(__name__) + + +class ResolutionStrategy(Enum): + """Strategies for resolving multiple annotations per instance.""" + MAJORITY_VOTE = "majority_vote" + RANDOM = "random" + CONSENSUS = "consensus" + WEIGHTED_AVERAGE = "weighted_average" + + +# --------------------------------------------------------------------------- +# SentenceTransformerVectorizer +# --------------------------------------------------------------------------- + +class SentenceTransformerVectorizer: + """sklearn-compatible wrapper for sentence-transformers. + + Uses dense embeddings from pre-trained transformer models instead of + bag-of-words features. Produces 384-dim vectors (for default model) + that capture semantic meaning, enabling better classification with + fewer training examples. + + The ``sentence-transformers`` package is an **optional** dependency and + is only imported when this vectorizer is actually used. + """ + + def __init__(self, model_name: str = "all-MiniLM-L6-v2"): + self.model_name = model_name + self._model = None + + def fit(self, X, y=None): + from sentence_transformers import SentenceTransformer + self._model = SentenceTransformer(self.model_name) + return self + + def transform(self, X): + if self._model is None: + raise RuntimeError("SentenceTransformerVectorizer has not been fitted yet") + return self._model.encode(list(X), show_progress_bar=False) + + def fit_transform(self, X, y=None): + self.fit(X, y) + return self.transform(X) + + +# --------------------------------------------------------------------------- +# Query Strategies +# --------------------------------------------------------------------------- + +class QueryStrategy(ABC): + """Base class for active learning query strategies.""" + + @abstractmethod + def rank(self, texts: List[str], model, vectorizer, + annotated_texts: Optional[List[str]] = None) -> List[Tuple[int, float]]: + """Return list of (index, score) sorted by selection priority (highest first).""" + + +class UncertaintySampling(QueryStrategy): + """Select instances where classifier is least confident. + + Selects x* = argmax_x (1 - max_y P(y|x)), i.e., instances where the + model's best guess has lowest confidence. + """ + + def rank(self, texts, model, vectorizer, annotated_texts=None): + try: + features = vectorizer.transform(texts) + probas = model.predict_proba(features) + # Score = 1 - max_prob (higher = more uncertain = higher priority) + scores = 1.0 - np.max(probas, axis=1) + ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"UncertaintySampling failed: {e}") + return [(i, 0.5) for i in range(len(texts))] + + +class DiversitySampling(QueryStrategy): + """Select instances that maximize feature-space coverage. + + Uses cosine distance from already-annotated instances in the vectorized + feature space. Ensures the training set covers the full data distribution + rather than over-sampling one region. + """ + + def rank(self, texts, model, vectorizer, annotated_texts=None): + from sklearn.metrics.pairwise import cosine_distances + + try: + features = vectorizer.transform(texts) + if hasattr(features, 'toarray'): + features = features.toarray() + + if annotated_texts: + annotated_features = vectorizer.transform(annotated_texts) + if hasattr(annotated_features, 'toarray'): + annotated_features = annotated_features.toarray() + # Score = min cosine distance to any annotated instance + distances = cosine_distances(features, annotated_features) + scores = np.min(distances, axis=1) + else: + # No annotated texts yet: use distance from centroid + centroid = np.mean(features, axis=0, keepdims=True) + scores = cosine_distances(features, centroid).ravel() + + ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"DiversitySampling failed: {e}") + return [(i, 0.5) for i in range(len(texts))] + + +class BadgeStrategy(QueryStrategy): + """BADGE approximation: uncertainty-weighted diversity. + + Inspired by Ash et al. (2020) [Ref 1]. Full BADGE uses gradient embeddings + from neural networks. Our approximation: + 1. Weight feature vectors by (1 - max_prob) as uncertainty proxy + 2. Run k-means++ initialization on weighted vectors to select + diverse-uncertain instances. + """ + + def rank(self, texts, model, vectorizer, annotated_texts=None): + try: + features = vectorizer.transform(texts) + if hasattr(features, 'toarray'): + features = features.toarray() + + probas = model.predict_proba(features) + uncertainty = 1.0 - np.max(probas, axis=1) + + # Weight features by uncertainty + weighted = features * uncertainty[:, np.newaxis] + + # Use k-means++ initialization to select diverse-uncertain points + from sklearn.cluster import kmeans_plusplus + n_clusters = min(len(texts), max(1, len(texts) // 2)) + _, indices = kmeans_plusplus(weighted, n_clusters=n_clusters, + random_state=42) + + # Build score: selected centroids get highest scores + scores = np.zeros(len(texts)) + for rank_pos, idx in enumerate(indices): + scores[idx] = len(indices) - rank_pos # highest for first-selected + + # For non-selected, use uncertainty as tiebreaker + for i in range(len(texts)): + if scores[i] == 0: + scores[i] = uncertainty[i] * 0.01 + + ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"BadgeStrategy failed, falling back to uncertainty: {e}") + return UncertaintySampling().rank(texts, model, vectorizer, annotated_texts) + + +class BaldStrategy(QueryStrategy): + """BALD: Bayesian Active Learning by Disagreement. + + Based on Houlsby et al. (2011) [Ref 2]. Trains an ensemble of classifiers + with different random seeds/bootstrap samples. Selects instances with + highest mutual information: H[y|x] - E_theta[H[y|x,theta]], i.e., + where the ensemble disagrees most. + """ + + def __init__(self, n_estimators: int = 5, bootstrap_fraction: float = 0.8): + self.n_estimators = n_estimators + self.bootstrap_fraction = bootstrap_fraction + + def rank(self, texts, model, vectorizer, annotated_texts=None): + try: + features = vectorizer.transform(texts) + if hasattr(features, 'toarray'): + features = features.toarray() + + probas = model.predict_proba(features) + # Average entropy + avg_proba = probas + entropy_avg = -np.sum(avg_proba * np.log(avg_proba + 1e-10), axis=1) + + # For a single model, we approximate BALD by using dropout-like noise + # or by comparing with uniform. Since we store the ensemble models + # on the manager, we just use the single model's entropy here and + # the ensemble version is handled in ActiveLearningManager._train_bald_ensemble + scores = entropy_avg + ranked = sorted(enumerate(scores), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"BaldStrategy failed: {e}") + return [(i, 0.5) for i in range(len(texts))] + + def rank_with_ensemble(self, texts, ensemble_models, vectorizer): + """Rank using actual ensemble disagreement (mutual information).""" + try: + features = vectorizer.transform(texts) + if hasattr(features, 'toarray'): + features = features.toarray() + + all_probas = [] + for m in ensemble_models: + all_probas.append(m.predict_proba(features)) + + all_probas = np.array(all_probas) # (n_estimators, n_samples, n_classes) + + # Mean prediction across ensemble + mean_proba = np.mean(all_probas, axis=0) # (n_samples, n_classes) + + # H[y|x] - entropy of mean prediction + entropy_mean = -np.sum(mean_proba * np.log(mean_proba + 1e-10), axis=1) + + # E_theta[H[y|x,theta]] - mean of individual entropies + individual_entropies = -np.sum(all_probas * np.log(all_probas + 1e-10), axis=2) + mean_entropy = np.mean(individual_entropies, axis=0) + + # Mutual information = H[y|x] - E[H[y|x,theta]] + mutual_info = entropy_mean - mean_entropy + + ranked = sorted(enumerate(mutual_info), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"BaldStrategy ensemble ranking failed: {e}") + return [(i, 0.5) for i in range(len(texts))] + + +class HybridStrategy(QueryStrategy): + """Weighted combination of uncertainty and diversity scores. + + Combines strategies with configurable weights. Default: 0.7 uncertainty + + 0.3 diversity. + """ + + def __init__(self, weights: Optional[Dict[str, float]] = None): + self.weights = weights or {"uncertainty": 0.7, "diversity": 0.3} + + def rank(self, texts, model, vectorizer, annotated_texts=None): + try: + strategies = {} + if self.weights.get("uncertainty", 0) > 0: + strategies["uncertainty"] = UncertaintySampling() + if self.weights.get("diversity", 0) > 0: + strategies["diversity"] = DiversitySampling() + + # Collect raw scores from each strategy + all_scores = {} + for name, strategy in strategies.items(): + rankings = strategy.rank(texts, model, vectorizer, annotated_texts) + score_map = {idx: score for idx, score in rankings} + all_scores[name] = score_map + + # Normalize each strategy's scores to [0, 1] + for name in all_scores: + vals = list(all_scores[name].values()) + min_val, max_val = min(vals), max(vals) + rng = max_val - min_val if max_val > min_val else 1.0 + all_scores[name] = { + idx: (s - min_val) / rng for idx, s in all_scores[name].items() + } + + # Weighted combination + combined = {} + for i in range(len(texts)): + combined[i] = sum( + self.weights.get(name, 0) * all_scores.get(name, {}).get(i, 0) + for name in self.weights + ) + + ranked = sorted(combined.items(), key=lambda x: x[1], reverse=True) + return ranked + except Exception as e: + logger.warning(f"HybridStrategy failed: {e}") + return UncertaintySampling().rank(texts, model, vectorizer, annotated_texts) + + +# Strategy registry +STRATEGY_REGISTRY = { + "uncertainty": UncertaintySampling, + "diversity": DiversitySampling, + "badge": BadgeStrategy, + "bald": BaldStrategy, + "hybrid": HybridStrategy, +} + + +def create_query_strategy(config: 'ActiveLearningConfig') -> QueryStrategy: + """Create a query strategy from config.""" + strategy_name = config.query_strategy + if strategy_name == "hybrid": + return HybridStrategy(weights=config.hybrid_weights) + elif strategy_name == "bald": + params = config.bald_params + return BaldStrategy( + n_estimators=params.get("n_estimators", 5), + bootstrap_fraction=params.get("bootstrap_fraction", 0.8), + ) + elif strategy_name in STRATEGY_REGISTRY: + return STRATEGY_REGISTRY[strategy_name]() + else: + logger.warning(f"Unknown strategy '{strategy_name}', falling back to uncertainty") + return UncertaintySampling() + + +# --------------------------------------------------------------------------- +# ICLClassifier wrapper (Phase 5A) +# --------------------------------------------------------------------------- + +class ICLClassifier: + """Wraps ICLLabeler as an sklearn-compatible classifier for ensemble use. + + Enables combining LLM-based ICL predictions with traditional classifier + predictions in a hybrid ensemble for active learning scoring. + """ + + def __init__(self, icl_labeler, schema_name: str, label_names: List[str]): + self.icl_labeler = icl_labeler + self.schema_name = schema_name + self.label_names = label_names + self.classes_ = np.array(label_names) + + def predict_proba(self, texts: List[str]) -> np.ndarray: + """Get label probabilities from LLM via ICL.""" + n_classes = len(self.label_names) + probas = np.full((len(texts), n_classes), 1.0 / n_classes) + + for i, text in enumerate(texts): + try: + prediction = self.icl_labeler.label_instance( + instance_id=f"_al_query_{i}", + schema_name=self.schema_name, + instance_text=text, + ) + if prediction and prediction.predicted_label in self.label_names: + idx = self.label_names.index(prediction.predicted_label) + conf = prediction.confidence_score + # Distribute: conf to predicted label, (1-conf)/(n-1) to others + remaining = (1.0 - conf) / max(1, n_classes - 1) + probas[i] = remaining + probas[i, idx] = conf + except Exception: + pass # Keep uniform distribution + + return probas + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +@dataclass +class ActiveLearningConfig: + """Enhanced configuration for active learning.""" + enabled: bool = False + classifier_name: str = "sklearn.linear_model.LogisticRegression" + classifier_kwargs: Dict[str, Any] = None + vectorizer_name: str = "sklearn.feature_extraction.text.TfidfVectorizer" + vectorizer_kwargs: Dict[str, Any] = None + min_annotations_per_instance: int = 1 + min_instances_for_training: int = 10 + max_instances_to_reorder: Optional[int] = None + resolution_strategy: ResolutionStrategy = ResolutionStrategy.MAJORITY_VOTE + random_sample_percent: float = 0.2 + update_frequency: int = 5 + schema_names: List[str] = None + + # Classifier/vectorizer passthrough params (Phase 1C) + classifier_params: Dict[str, Any] = field(default_factory=dict) + vectorizer_params: Dict[str, Any] = field(default_factory=dict) + + # Probability calibration (Phase 1D) + calibrate_probabilities: bool = True + + # Query strategy (Phase 2) + query_strategy: str = "uncertainty" + hybrid_weights: Dict[str, float] = field( + default_factory=lambda: {"uncertainty": 0.7, "diversity": 0.3} + ) + bald_params: Dict[str, Any] = field( + default_factory=lambda: {"n_estimators": 5, "bootstrap_fraction": 0.8} + ) + + # Cold-start (Phase 3) + cold_start_strategy: str = "random" + cold_start_batch_size: int = 20 + + # ICL ensemble (Phase 5) + use_icl_ensemble: bool = False + icl_ensemble_params: Dict[str, Any] = field(default_factory=lambda: { + "initial_icl_weight": 0.7, + "final_icl_weight": 0.2, + "transition_instances": 100, + }) + + # Annotation routing (Phase 5D) + annotation_routing: bool = False + routing_thresholds: Dict[str, float] = field(default_factory=lambda: { + "auto_label_min_confidence": 0.9, + "show_suggestion_below": 0.5, + }) + verification_sample_rate: float = 0.2 + + # Database persistence + database_enabled: bool = False + database_config: Dict[str, Any] = None + + # Model persistence + model_persistence_enabled: bool = False + model_save_directory: Optional[str] = None + model_retention_count: int = 2 + + # LLM integration + llm_enabled: bool = False + llm_config: Dict[str, Any] = None + + def __post_init__(self): + if self.classifier_kwargs is None: + self.classifier_kwargs = {} + if self.vectorizer_kwargs is None: + self.vectorizer_kwargs = {} + if self.schema_names is None: + self.schema_names = [] + if self.database_config is None: + self.database_config = {} + if self.llm_config is None: + self.llm_config = {} + # Merge classifier_params into classifier_kwargs + if self.classifier_params: + self.classifier_kwargs.update(self.classifier_params) + # Merge vectorizer_params into vectorizer_kwargs + if self.vectorizer_params: + self.vectorizer_kwargs.update(self.vectorizer_params) + + +@dataclass +class TrainingMetrics: + """Metrics for a training run.""" + schema_name: str + training_time: float + accuracy: float + instance_count: int + timestamp: datetime + model_file_path: Optional[str] = None + confidence_distribution: Dict[str, float] = None + error_message: Optional[str] = None + + +class ModelPersistence: + """Handles model saving and loading with metadata.""" + + def __init__(self, save_directory: str, retention_count: int = 2): + self.save_directory = save_directory + self.retention_count = retention_count + self.logger = logging.getLogger(__name__) + + # Ensure directory exists + os.makedirs(save_directory, exist_ok=True) + + def save_model(self, model: Pipeline, schema_name: str, instance_count: int) -> str: + """Save a trained model with metadata.""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"{schema_name}_{instance_count}_{timestamp}.pkl" + filepath = os.path.join(self.save_directory, filename) + + try: + # Save the complete model (including vectorizer) + with open(filepath, 'wb') as f: + pickle.dump(model, f) + + self.logger.info(f"Saved model to {filepath}") + + # Clean up old models + self._cleanup_old_models(schema_name) + + return filepath + except Exception as e: + self.logger.error(f"Failed to save model: {e}") + raise + + def load_model(self, filepath: str) -> Optional[Pipeline]: + """Load a saved model.""" + try: + with open(filepath, 'rb') as f: + model = pickle.load(f) + + # TODO: Add schema validation here in the future + # This is a placeholder for future schema validation enhancement + + self.logger.info(f"Loaded model from {filepath}") + return model + except Exception as e: + self.logger.error(f"Failed to load model from {filepath}: {e}") + return None + + def _cleanup_old_models(self, schema_name: str): + """Clean up old models based on retention policy.""" + try: + # Find all model files for this schema + model_files = [] + + for filename in os.listdir(self.save_directory): + if filename.startswith(f"{schema_name}_") and filename.endswith(".pkl"): + filepath = os.path.join(self.save_directory, filename) + model_files.append((filepath, os.path.getmtime(filepath))) + + # Sort by modification time (newest first) + model_files.sort(key=lambda x: x[1], reverse=True) + + # Remove old models beyond retention count + for filepath, _ in model_files[self.retention_count:]: + try: + os.remove(filepath) + self.logger.info(f"Removed old model: {filepath}") + except Exception as e: + self.logger.warning(f"Failed to remove old model {filepath}: {e}") + + except Exception as e: + self.logger.error(f"Error during model cleanup: {e}") + + +class DatabaseStateManager: + """Manages database persistence for active learning state.""" + + def __init__(self, config: Dict[str, Any]): + self.config = config + self.logger = logging.getLogger(__name__) + self.connection = None + self._initialize_database() + + def _initialize_database(self): + """Initialize database connection and create tables.""" + try: + # Use the same database system as main Potato application + if self.config.get('type') == 'mysql': + self._init_mysql_connection() + else: + self._init_file_based_connection() + + self._create_tables() + self.logger.info("Active learning database initialized successfully") + except Exception as e: + self.logger.error(f"Failed to initialize database: {e}") + raise + + def _init_mysql_connection(self): + """Initialize MySQL connection.""" + # TODO: Implement MySQL connection + pass + + def _init_file_based_connection(self): + """Initialize file-based database connection.""" + # TODO: Implement file-based database + pass + + def _create_tables(self): + """Create database tables for active learning.""" + # TODO: Implement table creation + pass + + def save_training_metrics(self, metrics: TrainingMetrics): + """Save training metrics to database.""" + # TODO: Implement metrics saving + pass + + def get_training_history(self, schema_name: Optional[str] = None) -> List[TrainingMetrics]: + """Get training history from database.""" + # TODO: Implement history retrieval + return [] + + def save_schema_cycling_state(self, current_schema: str, schema_order: List[str]): + """Save current schema cycling state.""" + # TODO: Implement state saving + pass + + def get_schema_cycling_state(self) -> Tuple[str, List[str]]: + """Get current schema cycling state.""" + # TODO: Implement state retrieval + return "", [] + + +class SchemaCycler: + """Manages cycling through multiple annotation schemes.""" + + def __init__(self, schema_names: List[str], database_manager: Optional[DatabaseStateManager] = None): + self.schema_names = self._validate_schemas(schema_names) + self.database_manager = database_manager + self.current_index = 0 + self.logger = logging.getLogger(__name__) + self._lock = threading.Lock() + + # Load state from database if available + if self.database_manager: + self._load_state() + + def _validate_schemas(self, schema_names: List[str]) -> List[str]: + """Validate and filter schema names.""" + valid_schemas = [] + + for schema in schema_names: + # Exclude text and span annotation schemes + if schema in ['text', 'span']: + raise ValueError(f"Text and span annotation schemes are not supported for active learning: {schema}") + valid_schemas.append(schema) + + return valid_schemas + + def _load_state(self): + """Load cycling state from database.""" + try: + current_schema, schema_order = self.database_manager.get_schema_cycling_state() + with self._lock: + if current_schema in self.schema_names: + self.current_index = self.schema_names.index(current_schema) + except Exception as e: + self.logger.warning(f"Failed to load schema cycling state: {e}") + + def get_current_schema(self) -> Optional[str]: + """Get the current schema for training.""" + if not self.schema_names: + return None + with self._lock: + return self.schema_names[self.current_index] + + def advance_schema(self): + """Advance to the next schema in the cycle.""" + if not self.schema_names: + return + + with self._lock: + self.current_index = (self.current_index + 1) % len(self.schema_names) + current_schema = self.schema_names[self.current_index] + + # Save state to database if available + if self.database_manager: + try: + self.database_manager.save_schema_cycling_state( + current_schema, + self.schema_names + ) + except Exception as e: + self.logger.warning(f"Failed to save schema cycling state: {e}") + + def get_schema_order(self) -> List[str]: + """Get the current schema cycling order.""" + return self.schema_names.copy() + + +class ActiveLearningManager: + """ + Manages active learning operations including classifier training and instance reordering. + + This class provides thread-safe operations for: + - Training classifiers on annotated data + - Predicting confidence scores for unlabeled instances + - Reordering instances based on configurable query strategies + - Cold-start LLM-based instance selection + - ICL/classifier ensemble for improved ranking + - Noise-aware annotation routing + - Managing training state and progress + - Database persistence and model saving + """ + + def __init__(self, config: ActiveLearningConfig): + self.config = config + self.logger = logging.getLogger(__name__) + + # Thread safety + self._lock = threading.RLock() + self._training_queue = queue.Queue() + self._training_thread = None + self._stop_training = threading.Event() + + # State tracking + self._last_training_time = 0 + self._training_count = 0 + self._models = {} # schema_name -> trained_model + self._vectorizers = {} # schema_name -> fitted vectorizer + self._bald_ensembles = {} # schema_name -> list of classifiers + self._last_annotation_count = 0 + self._training_metrics = [] # List of TrainingMetrics + self._annotated_texts = {} # schema_name -> list of annotated texts + + # Query strategy + self._query_strategy = create_query_strategy(config) + + # Database and persistence + self.database_manager = None + self.model_persistence = None + self.schema_cycler = None + + # Initialize components + self._initialize_components() + + # Start training thread if enabled + if self.config.enabled: + self._start_training_thread() + + def _initialize_components(self): + """Initialize database, model persistence, and schema cycler.""" + # Initialize database manager if enabled + if self.config.database_enabled: + try: + self.database_manager = DatabaseStateManager(self.config.database_config) + except Exception as e: + self.logger.error(f"Failed to initialize database manager: {e}") + # Continue without database persistence + + # Initialize model persistence if enabled + if self.config.model_persistence_enabled and self.config.model_save_directory: + try: + self.model_persistence = ModelPersistence( + self.config.model_save_directory, + self.config.model_retention_count + ) + except Exception as e: + self.logger.error(f"Failed to initialize model persistence: {e}") + # Continue without model persistence + + # Initialize schema cycler + try: + self.schema_cycler = SchemaCycler(self.config.schema_names, self.database_manager) + except Exception as e: + self.logger.error(f"Failed to initialize schema cycler: {e}") + raise # Schema cycler is critical + + def _start_training_thread(self): + """Start the background training thread.""" + if self._training_thread is None or not self._training_thread.is_alive(): + self._training_thread = threading.Thread(target=self._training_worker, daemon=True) + self._training_thread.start() + self.logger.info("Active learning training thread started") + + def _training_worker(self): + """Background worker for training classifiers.""" + while not self._stop_training.is_set(): + try: + # Wait for training request + training_request = self._training_queue.get(timeout=1.0) + if training_request is None: # Shutdown signal + break + + self._perform_training() + self._training_queue.task_done() + + except queue.Empty: + continue + except Exception as e: + self.logger.error(f"Error in training worker: {e}") + + def _perform_training(self): + """Perform the actual classifier training.""" + with self._lock: + try: + self.logger.info("Starting active learning classifier training") + start_time = time.time() + + # Get current schema for training + current_schema = self.schema_cycler.get_current_schema() + if not current_schema: + self.logger.warning("No schema available for training") + return + + # Get current annotation state + item_manager = get_item_state_manager() + user_manager = get_user_state_manager() + + # Collect training data + training_data = self._collect_training_data(item_manager, user_manager, current_schema) + + if not training_data: + self.logger.warning(f"No training data available for schema {current_schema}") + # If in cold-start phase, try LLM-based reordering + if self.config.cold_start_strategy == "llm" and self.config.llm_enabled: + self._cold_start_reorder(item_manager) + return + + # Train classifier + model, metrics = self._train_classifier(training_data, current_schema) + + if model: + self._models[current_schema] = model + self._annotated_texts[current_schema] = training_data["texts"] + + # Save model if persistence is enabled + if self.model_persistence: + try: + model_path = self.model_persistence.save_model( + model, current_schema, len(training_data["texts"]) + ) + metrics.model_file_path = model_path + except Exception as e: + self.logger.error(f"Failed to save model: {e}") + + # Save metrics to database if available + if self.database_manager: + try: + self.database_manager.save_training_metrics(metrics) + except Exception as e: + self.logger.error(f"Failed to save metrics: {e}") + + # Reorder instances + self._reorder_instances(item_manager, current_schema) + + # Advance to next schema + self.schema_cycler.advance_schema() + + self._training_count += 1 + self._last_training_time = time.time() + + training_duration = time.time() - start_time + self.logger.info(f"Active learning training completed for schema {current_schema} " + f"(run #{self._training_count}, duration: {training_duration:.2f}s)") + else: + self.logger.warning(f"Failed to train model for schema {current_schema}") + # Try cold-start if not enough data + if (self.config.cold_start_strategy == "llm" + and self.config.llm_enabled + and len(training_data.get("texts", [])) < self.config.min_instances_for_training): + self._cold_start_reorder(item_manager) + + except Exception as e: + self.logger.error(f"Error during training: {e}") + # Continue without failing the entire system + + def _collect_training_data(self, item_manager: ItemStateManager, user_manager, schema_name: str) -> Dict: + """Collect training data for a specific schema.""" + training_data = {"texts": [], "labels": [], "instance_ids": []} + + # Get all user states + user_states = user_manager.get_all_users() + self.logger.debug(f"Found {len(user_states)} user states") + + # Collect annotations per instance + instance_annotations = defaultdict(list) + + for user_state in user_states: + user_annotations = user_state.get_all_annotations() + self.logger.debug(f"User {user_state.user_id} has {len(user_annotations)} annotations") + for instance_id, annotations in user_annotations.items(): + # Check if the schema exists in the labels section + if 'labels' in annotations: + labels_dict = annotations['labels'] + # Handle Label objects as keys + for label_obj, value in labels_dict.items(): + if hasattr(label_obj, 'get_schema') and label_obj.get_schema() == schema_name: + instance_annotations[instance_id].append({ + "label": label_obj.get_name(), + "value": value, + "user": user_state.user_id + }) + + self.logger.debug(f"Collected annotations for {len(instance_annotations)} instances") + + # Filter instances with sufficient annotations + for instance_id, annotations in instance_annotations.items(): + if len(annotations) >= self.config.min_annotations_per_instance: + # Resolve multiple annotations + resolved_label = self._resolve_annotations(annotations) + if resolved_label: + item = item_manager.get_item(instance_id) + if item: + text = item.get_text() + training_data["texts"].append(text) + training_data["labels"].append(resolved_label) + training_data["instance_ids"].append(instance_id) + + self.logger.debug(f"Training data collected: {len(training_data['texts'])} texts, {len(training_data['labels'])} labels") + return training_data + + def _resolve_annotations(self, annotations: List[Dict]) -> Optional[str]: + """Resolve multiple annotations using the configured strategy.""" + if not annotations: + return None + + if self.config.resolution_strategy == ResolutionStrategy.MAJORITY_VOTE: + return self._majority_vote(annotations) + elif self.config.resolution_strategy == ResolutionStrategy.RANDOM: + return self._random_selection(annotations) + elif self.config.resolution_strategy == ResolutionStrategy.CONSENSUS: + return self._consensus_resolution(annotations) + else: + return self._majority_vote(annotations) # Default fallback + + def _majority_vote(self, annotations: List[Dict]) -> str: + """Resolve annotations using majority vote with random tie-breaking.""" + label_counts = Counter(ann["label"] for ann in annotations) + max_count = max(label_counts.values()) + # Find all labels with the maximum count (handles ties) + tied_labels = [label for label, count in label_counts.items() if count == max_count] + # Break ties randomly + return random.choice(tied_labels) + + def _random_selection(self, annotations: List[Dict]) -> str: + """Resolve annotations by random selection.""" + return random.choice(annotations)["label"] + + def _consensus_resolution(self, annotations: List[Dict]) -> Optional[str]: + """Resolve annotations by consensus (all must agree).""" + labels = [ann["label"] for ann in annotations] + if len(set(labels)) == 1: + return labels[0] + return None + + def _train_classifier(self, training_data: Dict, schema_name: str) -> Tuple[Optional[Pipeline], TrainingMetrics]: + """Train a classifier for a specific schema.""" + start_time = time.time() + + if len(training_data["texts"]) < self.config.min_instances_for_training: + error_msg = f"Insufficient training data for schema {schema_name}: {len(training_data['texts'])} < {self.config.min_instances_for_training}" + self.logger.warning(error_msg) + return None, TrainingMetrics( + schema_name=schema_name, + training_time=time.time() - start_time, + accuracy=0.0, + instance_count=len(training_data["texts"]), + timestamp=datetime.now(), + error_message=error_msg + ) + + # Check for sufficient label diversity + unique_labels = set(training_data["labels"]) + if len(unique_labels) < 2: + error_msg = f"Insufficient label diversity for schema {schema_name}: {len(unique_labels)} unique labels" + self.logger.warning(error_msg) + return None, TrainingMetrics( + schema_name=schema_name, + training_time=time.time() - start_time, + accuracy=0.0, + instance_count=len(training_data["texts"]), + timestamp=datetime.now(), + error_message=error_msg + ) + + try: + # Create and train classifier + classifier = self._create_classifier() + vectorizer = self._create_vectorizer() + + pipeline = Pipeline([ + ("vectorizer", vectorizer), + ("classifier", classifier) + ]) + + pipeline.fit(training_data["texts"], training_data["labels"]) + + # Apply probability calibration if enabled + if self.config.calibrate_probabilities and hasattr(classifier, 'predict_proba'): + num_samples = len(training_data["texts"]) + if num_samples >= 5: + try: + from sklearn.calibration import CalibratedClassifierCV + cv_folds = min(3, num_samples // 2) + if cv_folds >= 2: + calibrated = CalibratedClassifierCV( + pipeline, cv=cv_folds, method='isotonic' + ) + calibrated.fit(training_data["texts"], training_data["labels"]) + pipeline = calibrated + self.logger.debug(f"Applied probability calibration with {cv_folds}-fold CV") + except Exception as e: + self.logger.warning(f"Calibration failed, using uncalibrated model: {e}") + + # Store vectorizer separately for strategy use + self._vectorizers[schema_name] = pipeline.named_steps.get("vectorizer", vectorizer) if hasattr(pipeline, 'named_steps') else vectorizer + + # Train BALD ensemble if needed + if self.config.query_strategy == "bald": + self._train_bald_ensemble(training_data, schema_name) + + # Calculate accuracy + predictions = pipeline.predict(training_data["texts"]) + accuracy = accuracy_score(training_data["labels"], predictions) + + # Calculate confidence distribution + confidence_distribution = self._calculate_confidence_distribution(pipeline, training_data["texts"]) + + training_time = time.time() - start_time + + metrics = TrainingMetrics( + schema_name=schema_name, + training_time=training_time, + accuracy=accuracy, + instance_count=len(training_data["texts"]), + timestamp=datetime.now(), + confidence_distribution=confidence_distribution + ) + + self.logger.info(f"Trained classifier for schema {schema_name} with {len(training_data['texts'])} instances, " + f"accuracy: {accuracy:.3f}, time: {training_time:.2f}s") + + return pipeline, metrics + + except Exception as e: + error_msg = f"Error training classifier for schema {schema_name}: {e}" + self.logger.error(error_msg) + return None, TrainingMetrics( + schema_name=schema_name, + training_time=time.time() - start_time, + accuracy=0.0, + instance_count=len(training_data["texts"]), + timestamp=datetime.now(), + error_message=error_msg + ) + + def _train_bald_ensemble(self, training_data: Dict, schema_name: str): + """Train an ensemble of classifiers for BALD strategy.""" + params = self.config.bald_params + n_estimators = params.get("n_estimators", 5) + bootstrap_fraction = params.get("bootstrap_fraction", 0.8) + + texts = training_data["texts"] + labels = training_data["labels"] + n_samples = len(texts) + bootstrap_size = max(2, int(n_samples * bootstrap_fraction)) + + ensemble = [] + for i in range(n_estimators): + indices = np.random.choice(n_samples, size=bootstrap_size, replace=True) + boot_texts = [texts[j] for j in indices] + boot_labels = [labels[j] for j in indices] + + # Need at least 2 classes + if len(set(boot_labels)) < 2: + continue + + clf = self._create_classifier() + vec = self._create_vectorizer() + pipe = Pipeline([("vectorizer", vec), ("classifier", clf)]) + pipe.fit(boot_texts, boot_labels) + ensemble.append(pipe) + + if ensemble: + self._bald_ensembles[schema_name] = ensemble + self.logger.info(f"Trained BALD ensemble with {len(ensemble)} models for {schema_name}") + + def _calculate_confidence_distribution(self, pipeline, texts: List[str]) -> Dict[str, float]: + """Calculate confidence score distribution.""" + try: + probas = pipeline.predict_proba(texts) + max_confidences = np.max(probas, axis=1) + + # Create histogram bins + bins = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0] + hist, _ = np.histogram(max_confidences, bins=bins) + + # Convert to percentages + total = len(max_confidences) + distribution = {} + for i, count in enumerate(hist): + bin_label = f"{bins[i]:.1f}-{bins[i+1]:.1f}" + distribution[bin_label] = (count / total) * 100 if total > 0 else 0 + + return distribution + except Exception as e: + self.logger.warning(f"Failed to calculate confidence distribution: {e}") + return {} + + def _create_classifier(self): + """Create classifier instance based on configuration.""" + kwargs = dict(self.config.classifier_kwargs) + + if self.config.classifier_name == "sklearn.linear_model.LogisticRegression": + return LogisticRegression(**kwargs) + elif self.config.classifier_name == "sklearn.ensemble.RandomForestClassifier": + return RandomForestClassifier(**kwargs) + elif self.config.classifier_name == "sklearn.svm.SVC": + kwargs.setdefault("probability", True) + return SVC(**kwargs) + else: + # Try to import dynamically + try: + module_name, class_name = self.config.classifier_name.rsplit('.', 1) + module = __import__(module_name, fromlist=[class_name]) + classifier_class = getattr(module, class_name) + return classifier_class(**kwargs) + except Exception as e: + self.logger.error(f"Failed to create classifier {self.config.classifier_name}: {e}") + return LogisticRegression() # Fallback + + def _create_vectorizer(self): + """Create vectorizer instance based on configuration.""" + kwargs = dict(self.config.vectorizer_kwargs) + + if self.config.vectorizer_name == "sklearn.feature_extraction.text.CountVectorizer": + return CountVectorizer(**kwargs) + elif self.config.vectorizer_name == "sklearn.feature_extraction.text.TfidfVectorizer": + return TfidfVectorizer(**kwargs) + elif self.config.vectorizer_name == "sentence-transformers": + model_name = kwargs.pop("model_name", "all-MiniLM-L6-v2") + return SentenceTransformerVectorizer(model_name=model_name) + else: + # Try to import dynamically + try: + module_name, class_name = self.config.vectorizer_name.rsplit('.', 1) + module = __import__(module_name, fromlist=[class_name]) + vectorizer_class = getattr(module, class_name) + return vectorizer_class(**kwargs) + except Exception as e: + self.logger.error(f"Failed to create vectorizer {self.config.vectorizer_name}: {e}") + return TfidfVectorizer() # Fallback + + def _reorder_instances(self, item_manager: ItemStateManager, schema_name: str): + """Reorder instances based on the configured query strategy.""" + if schema_name not in self._models: + self.logger.warning(f"No trained model available for schema {schema_name}") + return + + # Get unlabeled instances + unlabeled_instances = [] + unlabeled_texts = [] + for instance_id in item_manager.get_instance_ids(): + if not item_manager.get_annotators_for_item(instance_id): + item = item_manager.get_item(instance_id) + if item: + unlabeled_instances.append(instance_id) + unlabeled_texts.append(item.get_text()) + + if not unlabeled_texts: + self.logger.info("No unlabeled instances to reorder") + return + + # Limit number of instances to process + if self.config.max_instances_to_reorder: + limit = self.config.max_instances_to_reorder + unlabeled_instances = unlabeled_instances[:limit] + unlabeled_texts = unlabeled_texts[:limit] + + model = self._models[schema_name] + annotated = self._annotated_texts.get(schema_name, []) + + # Get rankings from strategy + if (self.config.query_strategy == "bald" + and schema_name in self._bald_ensembles + and isinstance(self._query_strategy, BaldStrategy)): + vectorizer = self._vectorizers.get(schema_name) + if vectorizer: + rankings = self._query_strategy.rank_with_ensemble( + unlabeled_texts, self._bald_ensembles[schema_name], vectorizer + ) + else: + rankings = self._query_strategy.rank(unlabeled_texts, model, model, annotated) + else: + # Extract vectorizer and classifier from pipeline for strategy use + vectorizer = self._vectorizers.get(schema_name) + classifier = model + if vectorizer: + rankings = self._query_strategy.rank( + unlabeled_texts, classifier, vectorizer, annotated + ) + else: + # Fallback: use confidence scores directly + instance_scores = self._calculate_confidence_scores( + unlabeled_instances, item_manager, schema_name + ) + sorted_instances = sorted(instance_scores, key=lambda x: x[1]) + self._apply_reordering(sorted_instances, item_manager) + return + + # ICL ensemble blending (Phase 5B) + if self.config.use_icl_ensemble: + rankings = self._blend_icl_scores( + rankings, unlabeled_texts, schema_name + ) + + # Map rankings back to instance IDs + sorted_instances = [ + (unlabeled_instances[idx], score) for idx, score in rankings + if idx < len(unlabeled_instances) + ] + + # Apply reordering with random sampling + self._apply_reordering(sorted_instances, item_manager) + + def _blend_icl_scores(self, rankings: List[Tuple[int, float]], + texts: List[str], schema_name: str) -> List[Tuple[int, float]]: + """Blend query strategy scores with ICL predictions.""" + try: + from potato.ai.icl_labeler import get_icl_labeler + icl_labeler = get_icl_labeler() + if icl_labeler is None or not icl_labeler.has_enough_examples(schema_name): + return rankings + + # Determine interpolation weight based on annotation count + params = self.config.icl_ensemble_params + initial_w = params.get("initial_icl_weight", 0.7) + final_w = params.get("final_icl_weight", 0.2) + transition = params.get("transition_instances", 100) + + annotated_count = len(self._annotated_texts.get(schema_name, [])) + progress = min(1.0, annotated_count / max(1, transition)) + icl_weight = initial_w + (final_w - initial_w) * progress + strategy_weight = 1.0 - icl_weight + + # Get ICL confidence for each text + icl_scores = {} + for idx, text in enumerate(texts): + try: + pred = icl_labeler.label_instance( + instance_id=f"_al_blend_{idx}", + schema_name=schema_name, + instance_text=text, + ) + if pred: + # Lower confidence = higher priority (more uncertain) + icl_scores[idx] = 1.0 - pred.confidence_score + else: + icl_scores[idx] = 0.5 + except Exception: + icl_scores[idx] = 0.5 + + # Normalize strategy scores + strategy_map = {idx: score for idx, score in rankings} + s_vals = list(strategy_map.values()) + s_min, s_max = min(s_vals), max(s_vals) + s_rng = s_max - s_min if s_max > s_min else 1.0 + + # Normalize ICL scores + i_vals = list(icl_scores.values()) + i_min, i_max = min(i_vals), max(i_vals) + i_rng = i_max - i_min if i_max > i_min else 1.0 + + blended = [] + for idx, score in rankings: + norm_s = (score - s_min) / s_rng + norm_i = (icl_scores.get(idx, 0.5) - i_min) / i_rng + combined = strategy_weight * norm_s + icl_weight * norm_i + blended.append((idx, combined)) + + blended.sort(key=lambda x: x[1], reverse=True) + return blended + + except ImportError: + return rankings + except Exception as e: + self.logger.warning(f"ICL blending failed: {e}") + return rankings + + def _cold_start_reorder(self, item_manager: ItemStateManager): + """LLM-based cold-start instance selection (Phase 3A). + + Based on Bayer et al. (2024) ActiveLLM approach. Before enough + annotations exist for classifier training, use LLM to estimate + which instances are most informative by finding those where LLM + confidence is moderate (on the decision boundary). + """ + try: + from potato.ai.llm_active_learning import create_llm_active_learning + + llm = create_llm_active_learning(self.config.llm_config) + + # Sample candidate instances + all_ids = list(item_manager.get_instance_ids()) + unannotated = [ + iid for iid in all_ids + if not item_manager.get_annotators_for_item(iid) + ] + + if not unannotated: + return + + batch_size = min(self.config.cold_start_batch_size, len(unannotated)) + candidates = random.sample(unannotated, batch_size) + + instances = [] + for iid in candidates: + item = item_manager.get_item(iid) + if item: + instances.append({"id": iid, "text": item.get_text()}) + + if not instances: + return + + # Get LLM predictions + schema_name = self.schema_cycler.get_current_schema() if self.schema_cycler else None + predictions = llm.predict_instances( + instances=instances, + annotation_instructions="Rate your confidence in labeling this text.", + schema_name=schema_name or "default", + label_options=["positive", "negative", "neutral"], + ) + + # Select instances with moderate confidence (decision boundary) + moderate = [] + other = [] + for pred in predictions: + if 0.4 <= pred.confidence_score <= 0.7: + moderate.append((pred.instance_id, pred.confidence_score)) + else: + other.append((pred.instance_id, pred.confidence_score)) + + # Moderate-confidence first, then others, interleaved with random + reordered = [iid for iid, _ in moderate] + [iid for iid, _ in other] + + # Add remaining unannotated instances not in the sample + sampled_set = set(candidates) + remaining = [iid for iid in unannotated if iid not in sampled_set] + random.shuffle(remaining) + reordered.extend(remaining) + + item_manager.reorder_instances(reordered) + self.logger.info(f"Cold-start LLM reordering: {len(moderate)} moderate-confidence, " + f"{len(other)} other, {len(remaining)} remaining") + + except Exception as e: + self.logger.warning(f"Cold-start LLM reordering failed: {e}") + + def _route_annotation(self, instance_id: str, instance_text: str, + schema_name: str) -> Dict[str, Any]: + """Noise-aware annotation routing (Phase 5D). + + Based on Yuan et al. (2024) NoiseAL approach. Routes instances + between LLM auto-labeling and human annotation based on LLM + confidence levels. + + Returns: + Dict with 'route' ('human'|'auto'), optional 'suggestion', + and optional 'auto_label'. + """ + if not self.config.annotation_routing: + return {"route": "human"} + + thresholds = self.config.routing_thresholds + auto_min = thresholds.get("auto_label_min_confidence", 0.9) + suggest_below = thresholds.get("show_suggestion_below", 0.5) + + try: + from potato.ai.icl_labeler import get_icl_labeler + icl_labeler = get_icl_labeler() + if icl_labeler is None or not icl_labeler.has_enough_examples(schema_name): + return {"route": "human"} + + prediction = icl_labeler.label_instance( + instance_id=instance_id, + schema_name=schema_name, + instance_text=instance_text, + ) + + if prediction is None: + return {"route": "human"} + + confidence = prediction.confidence_score + + if confidence >= auto_min: + # High confidence: auto-label with periodic verification + should_verify = random.random() < self.config.verification_sample_rate + return { + "route": "auto", + "auto_label": prediction.predicted_label, + "confidence": confidence, + "needs_verification": should_verify, + } + elif confidence < suggest_below: + # Low confidence: route to human with LLM suggestion + return { + "route": "human", + "suggestion": prediction.predicted_label, + "confidence": confidence, + } + else: + # Medium confidence: route to human (most informative) + return {"route": "human"} + + except ImportError: + return {"route": "human"} + except Exception as e: + self.logger.warning(f"Annotation routing failed for {instance_id}: {e}") + return {"route": "human"} + + def _calculate_confidence_scores(self, instance_ids: List[str], item_manager: ItemStateManager, schema_name: str) -> List[Tuple[str, float]]: + """Calculate confidence scores for instances.""" + instance_scores = [] + model = self._models[schema_name] + + for instance_id in instance_ids: + item = item_manager.get_item(instance_id) + if not item: + continue + + text = item.get_text() + + try: + # Get prediction probabilities + probas = model.predict_proba([text])[0] + confidence = np.max(probas) + instance_scores.append((instance_id, confidence)) + except Exception as e: + self.logger.warning(f"Error predicting for instance {instance_id}: {e}") + # Default to low confidence for failed predictions + instance_scores.append((instance_id, 0.1)) + + return instance_scores + + def _apply_reordering(self, sorted_instances: List[Tuple[str, float]], item_manager: ItemStateManager): + """Apply the new ordering to the item manager.""" + # Extract instance IDs in new order + new_order = [instance_id for instance_id, _ in sorted_instances] + + if not new_order: + return + + # Apply random sampling + random_count = int(len(new_order) * self.config.random_sample_percent) + if random_count > 0 and random_count <= len(new_order): + random_instances = random.sample(new_order, random_count) + else: + random_instances = [] + + # Interleave active learning and random instances + final_order = [] + al_idx = 0 + rand_idx = 0 + + while al_idx < len(new_order) or rand_idx < len(random_instances): + if al_idx < len(new_order): + final_order.append(new_order[al_idx]) + al_idx += 1 + if rand_idx < len(random_instances): + final_order.append(random_instances[rand_idx]) + rand_idx += 1 + + # Update item manager ordering + item_manager.reorder_instances(final_order) + self.logger.info(f"Reordered {len(final_order)} instances") + + def check_and_trigger_training(self): + """Check if training should be triggered and queue it if needed.""" + if not self.config.enabled: + self.logger.debug("Active learning is disabled") + return + + with self._lock: + # Count current annotations + user_manager = get_user_state_manager() + current_annotation_count = sum( + len(user_state.get_all_annotations()) + for user_state in user_manager.get_all_users() + ) + + self.logger.debug(f"Current annotation count: {current_annotation_count}, last count: {self._last_annotation_count}, update_frequency: {self.config.update_frequency}") + + # Check if we should trigger training + if (current_annotation_count - self._last_annotation_count) >= self.config.update_frequency: + self._training_queue.put("train") + self._last_annotation_count = current_annotation_count + self.logger.info(f"Queued active learning training (annotations: {current_annotation_count})") + else: + self.logger.debug("Not enough new annotations to trigger training") + + def force_training(self): + """Force immediate training (for testing purposes).""" + if not self.config.enabled: + self.logger.debug("Active learning is disabled") + return + + self.logger.info("Forcing immediate active learning training") + self._training_queue.put("train") + + def get_stats(self) -> Dict[str, Any]: + """Get active learning statistics.""" + with self._lock: + stats = { + "enabled": self.config.enabled, + "training_count": self._training_count, + "last_training_time": self._last_training_time, + "models_trained": list(self._models.keys()), + "current_schema": self.schema_cycler.get_current_schema() if self.schema_cycler else None, + "schema_order": self.schema_cycler.get_schema_order() if self.schema_cycler else [], + "database_enabled": self.config.database_enabled, + "model_persistence_enabled": self.config.model_persistence_enabled, + "llm_enabled": self.config.llm_enabled, + "query_strategy": self.config.query_strategy, + "calibrate_probabilities": self.config.calibrate_probabilities, + "cold_start_strategy": self.config.cold_start_strategy, + "use_icl_ensemble": self.config.use_icl_ensemble, + "annotation_routing": self.config.annotation_routing, + } + + # Add training metrics if available + if self.database_manager: + try: + stats["training_history"] = [ + asdict(metrics) for metrics in self.database_manager.get_training_history() + ] + except Exception as e: + self.logger.warning(f"Failed to get training history: {e}") + stats["training_history"] = [] + + return stats + + def shutdown(self): + """Shutdown the active learning manager.""" + self._stop_training.set() + if self._training_thread and self._training_thread.is_alive(): + self._training_queue.put(None) # Shutdown signal + self._training_thread.join(timeout=5.0) + self.logger.info("Active learning manager shutdown complete") + + +# Global singleton instance +ACTIVE_LEARNING_MANAGER = None + + +def parse_active_learning_config(config_data: Dict[str, Any]) -> Optional[ActiveLearningConfig]: + """Build an ``ActiveLearningConfig`` from a Potato project config dict. + + Returns None when active learning is not enabled. Maps the keys under the + ``active_learning:`` section onto the dataclass fields (unknown keys are + ignored), and defaults ``schema_names`` to the project's labelable + annotation schemes when not given. + """ + al_dict = (config_data or {}).get("active_learning", {}) or {} + if not al_dict.get("enabled"): + return None + + valid_fields = {f.name for f in dataclasses.fields(ActiveLearningConfig)} + kwargs = {k: v for k, v in al_dict.items() if k in valid_fields} + + # Honor the nested `active_learning.llm:` block (LLM cold-start / ICL). + # The dataclass uses flat fields (llm_enabled / llm_config), so translate. + llm_block = al_dict.get("llm") + if isinstance(llm_block, dict): + kwargs.setdefault("llm_enabled", bool(llm_block.get("enabled", False))) + kwargs.setdefault("llm_config", llm_block) + + # YAML parses sequences as lists, but sklearn's vectorizers require a tuple + # for ngram_range (e.g. (1, 2)). Coerce it so training doesn't fail. + vec_params = kwargs.get("vectorizer_params") + if isinstance(vec_params, dict) and isinstance(vec_params.get("ngram_range"), list): + vec_params = dict(vec_params) + vec_params["ngram_range"] = tuple(vec_params["ngram_range"]) + kwargs["vectorizer_params"] = vec_params + + # resolution_strategy may arrive as a string; coerce to the enum. + rs = kwargs.get("resolution_strategy") + if isinstance(rs, str): + try: + kwargs["resolution_strategy"] = ResolutionStrategy(rs) + except ValueError: + kwargs.pop("resolution_strategy", None) + + # Default schema_names to the labelable schemes in the project. + if not kwargs.get("schema_names"): + schemes = config_data.get("annotation_schemes", []) or [] + kwargs["schema_names"] = [ + s.get("name") for s in schemes + if s.get("name") and s.get("annotation_type") in ( + "radio", "multiselect", "likert", "select" + ) + ] + + return ActiveLearningConfig(**kwargs) + + +def init_active_learning_manager(config: ActiveLearningConfig) -> ActiveLearningManager: + """Initialize the global active learning manager.""" + global ACTIVE_LEARNING_MANAGER + + if ACTIVE_LEARNING_MANAGER is None: + ACTIVE_LEARNING_MANAGER = ActiveLearningManager(config) + + return ACTIVE_LEARNING_MANAGER + + +def get_active_learning_manager() -> Optional[ActiveLearningManager]: + """Get the global active learning manager.""" + return ACTIVE_LEARNING_MANAGER + + +def clear_active_learning_manager(): + """Clear the global active learning manager (for testing).""" + global ACTIVE_LEARNING_MANAGER + if ACTIVE_LEARNING_MANAGER: + ACTIVE_LEARNING_MANAGER.shutdown() + ACTIVE_LEARNING_MANAGER = None diff --git a/potato/adjudication.py b/potato/adjudication.py new file mode 100644 index 0000000000000000000000000000000000000000..afc731915162ef94a69756d044a597bc85ae16d4 --- /dev/null +++ b/potato/adjudication.py @@ -0,0 +1,1224 @@ +""" +Adjudication Module + +This module provides a comprehensive adjudication system where designated users +review items with multiple annotations, resolve disagreements, and produce +gold-standard final decisions. + +Adjudication is NOT a phase โ€” it's a parallel workflow accessible via a dedicated +/adjudicate route, available to users with adjudicator privileges. This avoids +disrupting the existing phase progression system. + +Key Components: +- AdjudicationConfig: Configuration dataclass for adjudication settings +- AdjudicationItem: Represents an item eligible for adjudication with all annotations +- AdjudicationDecision: Represents an adjudicator's final decision on an item +- AdjudicationManager: Singleton manager for the adjudication workflow + +The workflow: +1. Annotators complete annotations via /annotate (existing workflow) +2. AdjudicationManager monitors annotation counts and agreement +3. Items are flagged when criteria are met (min annotations, low agreement) +4. Adjudicators review items via /adjudicate and submit decisions +5. Final dataset CLI merges unanimous + adjudicated decisions +""" + +import json +import logging +import math +import os +import threading +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from typing import Dict, List, Optional, Any, Set + +logger = logging.getLogger(__name__) + +# Singleton instance +_ADJUDICATION_MANAGER = None +_ADJUDICATION_LOCK = threading.Lock() + + +@dataclass +class AdjudicationConfig: + """Configuration for adjudication features.""" + enabled: bool = False + adjudicator_users: List[str] = field(default_factory=list) + + # Trigger criteria + min_annotations: int = 2 + require_fully_annotated: bool = False + agreement_threshold: float = 0.75 + show_all_items: bool = False + + # Display options + show_annotator_names: bool = True + show_timing_data: bool = True + show_agreement_scores: bool = True + fast_decision_warning_ms: int = 2000 + + # Adjudicator metadata fields + require_confidence: bool = True + require_notes_on_override: bool = False + error_taxonomy: List[str] = field(default_factory=lambda: [ + "ambiguous_text", "guideline_gap", "annotator_error", + "edge_case", "subjective_disagreement", "other" + ]) + + # Similarity (Phase 3, optional) + similarity_enabled: bool = False + similarity_model: str = "all-MiniLM-L6-v2" + similarity_top_k: int = 5 + similarity_precompute: bool = True + + # Output + output_subdir: str = "adjudication" + + +@dataclass +class AdjudicationItem: + """Represents an item eligible for adjudication with all annotator data.""" + instance_id: str + annotations: Dict[str, Dict[str, Any]] # user_id -> {schema: {label: value}} + span_annotations: Dict[str, List[Dict]] # user_id -> [span_dict, ...] + behavioral_data: Dict[str, Dict] # user_id -> {total_time_ms, ...} + agreement_scores: Dict[str, float] # schema_name -> agreement score + overall_agreement: float + num_annotators: int + status: str = "pending" # pending, in_progress, completed, skipped + assigned_adjudicator: Optional[str] = None + mace_predictions: Dict[str, Any] = field(default_factory=dict) # schema -> predicted label + + def to_dict(self) -> Dict[str, Any]: + """Serialize to dictionary for JSON output.""" + result = { + "instance_id": self.instance_id, + "annotations": self.annotations, + "span_annotations": self.span_annotations, + "behavioral_data": self.behavioral_data, + "agreement_scores": self.agreement_scores, + "overall_agreement": self.overall_agreement, + "num_annotators": self.num_annotators, + "status": self.status, + "assigned_adjudicator": self.assigned_adjudicator, + } + if self.mace_predictions: + result["mace_predictions"] = self.mace_predictions + return result + + +@dataclass +class AdjudicationDecision: + """Represents an adjudicator's final decision on an item.""" + instance_id: str + adjudicator_id: str + timestamp: str # ISO format string + label_decisions: Dict[str, Any] # schema -> value + span_decisions: List[Dict] # list of span dicts + source: Dict[str, str] # schema -> "annotator_X" | "adjudicator" | "merged" + confidence: str # "high", "medium", "low" + notes: str + error_taxonomy: List[str] + guideline_update_flag: bool = False + guideline_update_notes: str = "" + time_spent_ms: int = 0 + + def to_dict(self) -> Dict[str, Any]: + """Serialize to dictionary for JSON output.""" + return { + "instance_id": self.instance_id, + "adjudicator_id": self.adjudicator_id, + "timestamp": self.timestamp, + "label_decisions": self.label_decisions, + "span_decisions": self.span_decisions, + "source": self.source, + "confidence": self.confidence, + "notes": self.notes, + "error_taxonomy": self.error_taxonomy, + "guideline_update_flag": self.guideline_update_flag, + "guideline_update_notes": self.guideline_update_notes, + "time_spent_ms": self.time_spent_ms, + } + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "AdjudicationDecision": + """Deserialize from dictionary.""" + return cls( + instance_id=d["instance_id"], + adjudicator_id=d["adjudicator_id"], + timestamp=d["timestamp"], + label_decisions=d.get("label_decisions", {}), + span_decisions=d.get("span_decisions", []), + source=d.get("source", {}), + confidence=d.get("confidence", "medium"), + notes=d.get("notes", ""), + error_taxonomy=d.get("error_taxonomy", []), + guideline_update_flag=d.get("guideline_update_flag", False), + guideline_update_notes=d.get("guideline_update_notes", ""), + time_spent_ms=d.get("time_spent_ms", 0), + ) + + +class AdjudicationManager: + """ + Manages the adjudication workflow including queue building, agreement + computation, decision storage, and final dataset generation. + + Follows the singleton pattern used by QualityControlManager. + """ + + def __init__(self, config: Dict[str, Any]): + """ + Initialize the adjudication manager. + + Args: + config: The full application configuration dictionary + """ + self.config = config + self.logger = logging.getLogger(__name__) + self._lock = threading.RLock() + + # Parse configuration + self.adj_config = self._parse_config(config) + + # Queue and decisions + self.queue: Dict[str, AdjudicationItem] = {} # instance_id -> AdjudicationItem + self.decisions: Dict[str, AdjudicationDecision] = {} # instance_id -> decision + self._queue_built = False + + # Load any previously saved decisions + self._load_decisions() + + # Initialize similarity engine (Phase 3) + self.similarity_engine = None + if self.adj_config.similarity_enabled: + from potato.similarity import init_similarity_engine + self.similarity_engine = init_similarity_engine(config, self.adj_config) + if (self.similarity_engine and self.similarity_engine.enabled + and self.adj_config.similarity_precompute): + self._precompute_similarities() + + self.logger.info( + f"AdjudicationManager initialized: enabled={self.adj_config.enabled}, " + f"adjudicators={self.adj_config.adjudicator_users}" + ) + + def _parse_config(self, config: Dict[str, Any]) -> AdjudicationConfig: + """Parse adjudication configuration from the main config.""" + adj = AdjudicationConfig() + + adj_config = config.get("adjudication", {}) + if not adj_config or not adj_config.get("enabled", False): + return adj + + adj.enabled = True + adj.adjudicator_users = adj_config.get("adjudicator_users", []) + adj.min_annotations = adj_config.get("min_annotations", 2) + adj.require_fully_annotated = adj_config.get("require_fully_annotated", False) + adj.agreement_threshold = adj_config.get("agreement_threshold", 0.75) + adj.show_all_items = adj_config.get("show_all_items", False) + adj.show_annotator_names = adj_config.get("show_annotator_names", True) + adj.show_timing_data = adj_config.get("show_timing_data", True) + adj.show_agreement_scores = adj_config.get("show_agreement_scores", True) + adj.fast_decision_warning_ms = adj_config.get("fast_decision_warning_ms", 2000) + adj.require_confidence = adj_config.get("require_confidence", True) + adj.require_notes_on_override = adj_config.get("require_notes_on_override", False) + + if "error_taxonomy" in adj_config: + adj.error_taxonomy = adj_config["error_taxonomy"] + + # Similarity settings + sim_config = adj_config.get("similarity", {}) + if sim_config.get("enabled", False): + adj.similarity_enabled = True + adj.similarity_model = sim_config.get("model", "all-MiniLM-L6-v2") + adj.similarity_top_k = sim_config.get("top_k", 5) + adj.similarity_precompute = sim_config.get("precompute_on_start", True) + + adj.output_subdir = adj_config.get("output_subdir", "adjudication") + + return adj + + def is_adjudicator(self, username: str) -> bool: + """Check if a user is an authorized adjudicator.""" + if not self.adj_config.enabled: + return False + return username in self.adj_config.adjudicator_users + + def build_queue(self) -> List[AdjudicationItem]: + """ + Scan all user annotations and build the adjudication queue. + + Items become eligible when they have enough annotations and + agreement is below the threshold. + + Returns: + List of AdjudicationItem objects + """ + from potato.user_state_management import get_user_state_manager + from potato.item_state_management import get_item_state_manager + + with self._lock: + usm = get_user_state_manager() + ism = get_item_state_manager() + + # Get all annotation schemes from config + annotation_schemes = self.config.get("annotation_schemes", []) + scheme_names = [s.get("name", "") for s in annotation_schemes] + + # Iterate over all items + for instance_id, item in ism.instance_id_to_instance.items(): + instance_id_str = str(instance_id) + + # Skip if already decided + if instance_id_str in self.decisions: + if instance_id_str not in self.queue: + continue + # Mark as completed if decision exists + self.queue[instance_id_str].status = "completed" + continue + + # Get all annotators for this item + annotators = ism.instance_annotators.get(instance_id, set()) + # Filter out adjudicators from annotator list + annotators = { + u for u in annotators + if u not in self.adj_config.adjudicator_users + } + + if len(annotators) < self.adj_config.min_annotations: + continue + + # Check if we require fully annotated items + if self.adj_config.require_fully_annotated: + max_per_item = ism.max_annotations_per_item + if max_per_item > 0 and len(annotators) < max_per_item: + continue + + # Collect annotations from all annotators + item_annotations = {} + item_spans = {} + item_behavioral = {} + + for user_id in annotators: + user_state = usm.get_user_state(user_id) + if not user_state: + continue + + # Get label annotations + label_annots = user_state.instance_id_to_label_to_value.get( + instance_id_str, {} + ) + if label_annots: + item_annotations[user_id] = self._serialize_labels(label_annots) + + # Get span annotations + span_annots = user_state.instance_id_to_span_to_value.get( + instance_id_str, {} + ) + if span_annots: + item_spans[user_id] = self._serialize_spans(span_annots) + + # Get behavioral data + bd = user_state.instance_id_to_behavioral_data.get( + instance_id_str, {} + ) + if bd: + item_behavioral[user_id] = self._serialize_behavioral(bd) + + if not item_annotations and not item_spans: + continue + + # Compute agreement scores + agreement_scores = self._compute_agreement( + item_annotations, scheme_names + ) + overall = self._compute_overall_agreement(agreement_scores) + + # Filter by agreement threshold + if not self.adj_config.show_all_items: + if overall >= self.adj_config.agreement_threshold: + continue + + # Preserve existing status if already in queue + existing = self.queue.get(instance_id_str) + status = existing.status if existing else "pending" + assigned = existing.assigned_adjudicator if existing else None + + # Enrich with MACE predictions if available + mace_preds = {} + try: + from potato.mace_manager import get_mace_manager + mace_mgr = get_mace_manager() + if mace_mgr and mace_mgr.results: + for sname in scheme_names: + pred = mace_mgr.get_prediction(instance_id_str, sname) + if pred is not None: + mace_preds[sname] = pred + except Exception: + pass # MACE is optional + + self.queue[instance_id_str] = AdjudicationItem( + instance_id=instance_id_str, + annotations=item_annotations, + span_annotations=item_spans, + behavioral_data=item_behavioral, + agreement_scores=agreement_scores, + overall_agreement=overall, + num_annotators=len(annotators), + status=status, + assigned_adjudicator=assigned, + mace_predictions=mace_preds, + ) + + self._queue_built = True + return list(self.queue.values()) + + def try_enqueue_item(self, instance_id: str) -> bool: + """ + Evaluate a single item and, if it qualifies, add it to the queue. + + Called when an overlap-sample item saturates so that low-agreement + items show up in the adjudication queue without needing a full + ``build_queue()`` rescan. Returns True if the item ended up in the + queue, False otherwise. + """ + if not self.adj_config.enabled: + return False + + from potato.user_state_management import get_user_state_manager + from potato.item_state_management import get_item_state_manager + + usm = get_user_state_manager() + ism = get_item_state_manager() + if usm is None or ism is None: + return False + + with self._lock: + instance_id_str = str(instance_id) + if instance_id_str in self.decisions: + return False + item = ism.instance_id_to_instance.get(instance_id) + if item is None: + return False + + annotators = { + u for u in ism.instance_annotators.get(instance_id, set()) + if u not in self.adj_config.adjudicator_users + } + if len(annotators) < self.adj_config.min_annotations: + return False + + scheme_names = [s.get("name", "") for s in self.config.get("annotation_schemes", [])] + item_annotations: Dict[str, Any] = {} + item_spans: Dict[str, Any] = {} + item_behavioral: Dict[str, Any] = {} + for user_id in annotators: + ustate = usm.get_user_state(user_id) + if not ustate: + continue + la = ustate.instance_id_to_label_to_value.get(instance_id_str, {}) + if la: + item_annotations[user_id] = self._serialize_labels(la) + sa = ustate.instance_id_to_span_to_value.get(instance_id_str, {}) + if sa: + item_spans[user_id] = self._serialize_spans(sa) + bd = ustate.instance_id_to_behavioral_data.get(instance_id_str, {}) + if bd: + item_behavioral[user_id] = self._serialize_behavioral(bd) + + if not item_annotations and not item_spans: + return False + + agreement_scores = self._compute_agreement(item_annotations, scheme_names) + overall = self._compute_overall_agreement(agreement_scores) + if not self.adj_config.show_all_items: + if overall >= self.adj_config.agreement_threshold: + return False + + existing = self.queue.get(instance_id_str) + self.queue[instance_id_str] = AdjudicationItem( + instance_id=instance_id_str, + annotations=item_annotations, + span_annotations=item_spans, + behavioral_data=item_behavioral, + agreement_scores=agreement_scores, + overall_agreement=overall, + num_annotators=len(annotators), + status=existing.status if existing else "pending", + assigned_adjudicator=existing.assigned_adjudicator if existing else None, + ) + self.logger.info( + "Auto-routed item %s into adjudication queue (overall agreement=%.3f, " + "threshold=%.3f, annotators=%d)", + instance_id_str, overall, self.adj_config.agreement_threshold, len(annotators), + ) + return True + + def _serialize_labels(self, label_data: Dict) -> Dict[str, Any]: + """Convert label annotation data to serializable dict.""" + result = {} + for key, value in label_data.items(): + # Key might be a Label object or a string + if hasattr(key, 'get_schema'): + schema = key.get_schema() + name = key.get_name() + if schema not in result: + result[schema] = {} + result[schema][name] = value + elif isinstance(key, str): + result[key] = value + else: + result[str(key)] = value + return result + + def _serialize_spans(self, span_data: Dict) -> List[Dict]: + """Convert span annotation data to serializable list.""" + spans = [] + for key, value in span_data.items(): + if hasattr(key, 'get_schema'): + spans.append({ + "schema": key.get_schema(), + "name": key.get_name(), + "title": key.get_title() if hasattr(key, 'get_title') else "", + "start": key.get_start(), + "end": key.get_end(), + "id": key.get_id(), + "target_field": key.get_target_field() if hasattr(key, 'get_target_field') else None, + }) + elif isinstance(value, dict): + spans.append(value) + return spans + + def _serialize_behavioral(self, bd) -> Dict: + """Convert behavioral data to serializable dict.""" + if hasattr(bd, 'to_dict'): + return bd.to_dict() + elif isinstance(bd, dict): + return bd + return {} + + def _compute_agreement( + self, item_annotations: Dict[str, Dict], scheme_names: List[str] + ) -> Dict[str, float]: + """ + Compute per-schema agreement for an item. + + Uses simple percentage agreement (proportion of annotators who chose + the most common label). For more sophisticated metrics, simpledorff + can be used but requires multiple items. + + Returns: + Dict mapping schema_name to agreement score (0.0 - 1.0) + """ + agreement_scores = {} + + for schema in scheme_names: + values = [] + for user_id, user_annots in item_annotations.items(): + if schema in user_annots: + val = user_annots[schema] + # Normalize to comparable form + if isinstance(val, dict): + # Radio stores {label: label} (value is the label string) + # and multiselect stores {label: value/true}. A label is + # "selected" when its value is present/truthy. The old + # filter (v is True / == "true" / == 1) dropped radio's + # string value, collapsing every annotator to an empty + # frozenset -> a spurious 1.0 agreement even on total + # disagreement. + falsey = (False, None, "", "false", "False", 0, "0") + selected = frozenset( + k for k, v in val.items() if v not in falsey + ) + values.append(selected) + else: + values.append(val) + + if len(values) < 2: + continue + + # Compute pairwise agreement (percentage) + agree_count = 0 + total_pairs = 0 + for i in range(len(values)): + for j in range(i + 1, len(values)): + total_pairs += 1 + if values[i] == values[j]: + agree_count += 1 + + agreement_scores[schema] = ( + agree_count / total_pairs if total_pairs > 0 else 1.0 + ) + + return agreement_scores + + def _compute_overall_agreement(self, agreement_scores: Dict[str, float]) -> float: + """Compute overall agreement as the mean of per-schema scores.""" + if not agreement_scores: + return 1.0 + return sum(agreement_scores.values()) / len(agreement_scores) + + def get_queue( + self, + adjudicator_id: Optional[str] = None, + filter_status: Optional[str] = None, + ) -> List[AdjudicationItem]: + """ + Get the adjudication queue, optionally filtered by status. + + Args: + adjudicator_id: Optional adjudicator to filter by assignment + filter_status: Optional status filter ("pending", "completed", etc.) + + Returns: + List of AdjudicationItem objects + """ + with self._lock: + if not self._queue_built: + self.build_queue() + + items = list(self.queue.values()) + + if filter_status: + items = [i for i in items if i.status == filter_status] + + # Sort: pending first, then by agreement (lowest first) + items.sort(key=lambda x: ( + 0 if x.status == "pending" else 1 if x.status == "in_progress" else 2, + x.overall_agreement, + )) + + return items + + def get_item(self, instance_id: str) -> Optional[AdjudicationItem]: + """ + Get full item data for adjudication. + + Args: + instance_id: The instance ID to retrieve + + Returns: + AdjudicationItem or None if not in queue + """ + with self._lock: + if not self._queue_built: + self.build_queue() + return self.queue.get(str(instance_id)) + + def get_item_text(self, instance_id: str) -> str: + """Get the text content for an item.""" + from potato.item_state_management import get_item_state_manager + + ism = get_item_state_manager() + item = ism.instance_id_to_instance.get(instance_id) + if item: + # Use text_key from config if available + text_key = self.config.get("item_properties", {}).get("text_key", "text") + data = item.get_data() + if isinstance(data, dict) and text_key in data: + return data[text_key] + return item.get_text() + return "" + + def get_item_data(self, instance_id: str) -> Dict[str, Any]: + """Get the full raw data for an item.""" + from potato.item_state_management import get_item_state_manager + + ism = get_item_state_manager() + item = ism.instance_id_to_instance.get(instance_id) + if item: + data = item.get_data() + if isinstance(data, dict): + return data + return {"text": str(data)} + return {} + + def get_next_item(self, adjudicator_id: str) -> Optional[AdjudicationItem]: + """Get the next pending item for an adjudicator.""" + items = self.get_queue(filter_status="pending") + if items: + return items[0] + return None + + def skip_item(self, instance_id: str, adjudicator_id: str) -> bool: + """Mark an item as skipped.""" + with self._lock: + item = self.queue.get(str(instance_id)) + if item: + item.status = "skipped" + return True + return False + + def submit_decision(self, decision: AdjudicationDecision) -> bool: + """ + Submit an adjudication decision. + + Args: + decision: The AdjudicationDecision to save + + Returns: + True if successful + """ + with self._lock: + instance_id = str(decision.instance_id) + self.decisions[instance_id] = decision + + # Update queue status + if instance_id in self.queue: + self.queue[instance_id].status = "completed" + self.queue[instance_id].assigned_adjudicator = decision.adjudicator_id + + # Persist to disk + self._save_decisions() + + self.logger.info( + f"Adjudication decision saved for {instance_id} " + f"by {decision.adjudicator_id}" + ) + return True + + def get_stats(self) -> Dict[str, Any]: + """Get adjudication progress statistics.""" + with self._lock: + if not self._queue_built: + self.build_queue() + + total = len(self.queue) + completed = sum( + 1 for i in self.queue.values() if i.status == "completed" + ) + pending = sum( + 1 for i in self.queue.values() if i.status == "pending" + ) + skipped = sum( + 1 for i in self.queue.values() if i.status == "skipped" + ) + in_progress = sum( + 1 for i in self.queue.values() if i.status == "in_progress" + ) + + avg_agreement = 0.0 + if self.queue: + avg_agreement = sum( + i.overall_agreement for i in self.queue.values() + ) / len(self.queue) + + # Per-adjudicator stats + adjudicator_stats = defaultdict(lambda: {"completed": 0, "total_time_ms": 0}) + for decision in self.decisions.values(): + adj_id = decision.adjudicator_id + adjudicator_stats[adj_id]["completed"] += 1 + adjudicator_stats[adj_id]["total_time_ms"] += decision.time_spent_ms + + return { + "total": total, + "completed": completed, + "pending": pending, + "skipped": skipped, + "in_progress": in_progress, + "completion_rate": completed / total if total > 0 else 0.0, + "avg_agreement": avg_agreement, + "adjudicator_stats": dict(adjudicator_stats), + } + + def get_decision(self, instance_id: str) -> Optional[AdjudicationDecision]: + """Get the decision for an item, if one exists.""" + return self.decisions.get(str(instance_id)) + + # ------------------------------------------------------------------ + # Phase 3: Similarity integration + # ------------------------------------------------------------------ + + def _precompute_similarities(self) -> None: + """Precompute embeddings for all items in the item state manager.""" + if not self.similarity_engine or not self.similarity_engine.enabled: + return + + from potato.item_state_management import get_item_state_manager + + try: + ism = get_item_state_manager() + item_texts = {} + for instance_id, item in ism.instance_id_to_instance.items(): + text = self.get_item_text(str(instance_id)) + if text: + item_texts[str(instance_id)] = text + + if item_texts: + count = self.similarity_engine.precompute_embeddings(item_texts) + self.logger.info(f"Precomputed {count} similarity embeddings") + except Exception as e: + self.logger.error(f"Error precomputing similarities: {e}") + + def get_similar_items( + self, instance_id: str, include_metadata: bool = True + ) -> List[Dict[str, Any]]: + """ + Get similar items for a given instance, enriched with queue metadata. + + Args: + instance_id: The reference instance ID + include_metadata: Whether to include decision/consensus data + + Returns: + List of dicts with instance_id, similarity, and optional metadata + """ + if not self.similarity_engine or not self.similarity_engine.enabled: + return [] + + similar = self.similarity_engine.find_similar(instance_id) + results = [] + + for other_id, score in similar: + entry = { + "instance_id": other_id, + "similarity": round(score, 4), + "text_preview": self.similarity_engine.text_cache.get( + other_id, "" + ), + } + + if include_metadata: + queue_item = self.queue.get(other_id) + decision = self.decisions.get(other_id) + + entry["in_queue"] = queue_item is not None + entry["status"] = queue_item.status if queue_item else None + entry["overall_agreement"] = ( + queue_item.overall_agreement if queue_item else None + ) + + if decision: + entry["decision"] = "completed" + entry["consensus_label"] = None + else: + entry["decision"] = None + if queue_item: + entry["consensus_label"] = self._get_consensus_label( + queue_item + ) + else: + entry["consensus_label"] = None + + results.append(entry) + + return results + + def _get_consensus_label(self, item: AdjudicationItem) -> Optional[str]: + """ + Get the majority/consensus label for an item across the first schema. + + Args: + item: The AdjudicationItem + + Returns: + The most common label value as a string, or None + """ + if not item.annotations: + return None + + # Use the first schema that has values + for user_annots in item.annotations.values(): + for schema_name in user_annots: + # Collect all values for this schema + values = [] + for ua in item.annotations.values(): + val = ua.get(schema_name) + if val is not None: + if isinstance(val, dict): + # Multiselect: use frozenset representation + selected = sorted( + k for k, v in val.items() + if v is True or v == "true" or v == 1 + ) + values.append(", ".join(selected) if selected else str(val)) + else: + values.append(str(val)) + + if values: + counter = Counter(values) + return counter.most_common(1)[0][0] + + return None + + # ------------------------------------------------------------------ + # Phase 3: Behavioral signal analysis + # ------------------------------------------------------------------ + + def get_annotator_signals( + self, user_id: str, instance_id: str + ) -> Dict[str, Any]: + """ + Compute per-annotator quality signals for a specific item. + + Returns: + Dict with user_id, instance_id, flags list, and metrics dict + """ + flags = [] + metrics = {} + + instance_id = str(instance_id) + item = self.queue.get(instance_id) + if not item: + return {"user_id": user_id, "instance_id": instance_id, + "flags": [], "metrics": {}} + + # Get behavioral data for this user on this item + bd = item.behavioral_data.get(user_id, {}) + if hasattr(bd, 'to_dict'): + bd = bd.to_dict() + + total_time = bd.get("total_time_ms", 0) + metrics["total_time_ms"] = total_time + + # 1. Speed z-score vs user's typical time + user_times = self._get_user_times(user_id) + if len(user_times) >= 3 and total_time > 0: + mean_time = sum(user_times) / len(user_times) + std_time = math.sqrt( + sum((t - mean_time) ** 2 for t in user_times) / len(user_times) + ) + if std_time > 0: + z_score = (total_time - mean_time) / std_time + metrics["speed_z_score"] = round(z_score, 2) + if z_score < -2.0: + flags.append({ + "type": "unusually_fast", + "severity": "high", + "message": f"Annotation time ({total_time}ms) is {abs(z_score):.1f} std devs below average" + }) + + # 2. Fast decision warning + fast_threshold = self.adj_config.fast_decision_warning_ms + if fast_threshold > 0 and 0 < total_time < fast_threshold: + flags.append({ + "type": "fast_decision", + "severity": "medium", + "message": f"Decision made in {total_time}ms (below {fast_threshold}ms threshold)" + }) + + # 3. Annotation change count + raw_changes = bd.get("annotation_changes", []) + change_count = len(raw_changes) if isinstance(raw_changes, list) else int(raw_changes or 0) + metrics["annotation_changes"] = change_count + if change_count > 5: + flags.append({ + "type": "excessive_changes", + "severity": "medium", + "message": f"Made {change_count} annotation changes on this item" + }) + + # 4. Historical agreement rate with consensus + agreement_rate = self._compute_user_agreement_rate(user_id) + if agreement_rate is not None: + metrics["agreement_rate"] = round(agreement_rate, 3) + if agreement_rate < 0.4: + flags.append({ + "type": "low_agreement", + "severity": "high", + "message": f"Agreement rate with consensus: {agreement_rate:.0%}" + }) + + # 5. Similar item consistency + if self.similarity_engine and self.similarity_engine.enabled: + inconsistencies = self._check_similar_item_consistency( + user_id, instance_id + ) + metrics["similar_item_inconsistencies"] = inconsistencies + if inconsistencies > 0: + flags.append({ + "type": "similar_item_inconsistency", + "severity": "medium", + "message": f"Different label on {inconsistencies} similar item(s)" + }) + + return { + "user_id": user_id, + "instance_id": instance_id, + "flags": flags, + "metrics": metrics, + } + + def _get_user_times(self, user_id: str) -> List[float]: + """Collect all annotation times for a user across queue items.""" + times = [] + for item in self.queue.values(): + bd = item.behavioral_data.get(user_id, {}) + if hasattr(bd, 'to_dict'): + bd = bd.to_dict() + t = bd.get("total_time_ms", 0) + if t > 0: + times.append(t) + return times + + def _compute_user_agreement_rate(self, user_id: str) -> Optional[float]: + """ + Compute how often a user agrees with the consensus across all items. + + Returns: + Float 0-1 or None if insufficient data (needs >= 3 items) + """ + agree_count = 0 + total_count = 0 + + for item in self.queue.values(): + if user_id not in item.annotations: + continue + + consensus = self._get_consensus_label(item) + if consensus is None: + continue + + user_annots = item.annotations[user_id] + # Check the first schema + for schema_name, val in user_annots.items(): + if isinstance(val, dict): + selected = sorted( + k for k, v in val.items() + if v is True or v == "true" or v == 1 + ) + user_label = ", ".join(selected) if selected else str(val) + else: + user_label = str(val) + + if user_label == consensus: + agree_count += 1 + total_count += 1 + break # Only check first schema + + if total_count < 3: + return None + + return agree_count / total_count + + def _check_similar_item_consistency( + self, user_id: str, instance_id: str + ) -> int: + """ + Check if user's label on similar items (>0.8 similarity) is consistent. + + Returns: + Count of similar items where user's label differs + """ + if not self.similarity_engine: + return 0 + + similar = self.similarity_engine.find_similar(instance_id) + if not similar: + return 0 + + # Get user's label on the current item + item = self.queue.get(instance_id) + if not item or user_id not in item.annotations: + return 0 + + user_annots = item.annotations[user_id] + current_label = None + current_schema = None + for schema_name, val in user_annots.items(): + current_schema = schema_name + if isinstance(val, dict): + selected = sorted( + k for k, v in val.items() + if v is True or v == "true" or v == 1 + ) + current_label = ", ".join(selected) if selected else str(val) + else: + current_label = str(val) + break + + if current_label is None: + return 0 + + inconsistencies = 0 + for other_id, score in similar: + if score < 0.8: + break # Results are sorted by score desc + + other_item = self.queue.get(other_id) + if not other_item or user_id not in other_item.annotations: + continue + + other_annots = other_item.annotations[user_id] + other_val = other_annots.get(current_schema) + if other_val is None: + continue + + if isinstance(other_val, dict): + selected = sorted( + k for k, v in other_val.items() + if v is True or v == "true" or v == 1 + ) + other_label = ", ".join(selected) if selected else str(other_val) + else: + other_label = str(other_val) + + if other_label != current_label: + inconsistencies += 1 + + return inconsistencies + + def _get_output_dir(self) -> str: + """Get the adjudication output directory.""" + output_dir = self.config.get("output_annotation_dir", "annotation_output") + adj_dir = os.path.join(output_dir, self.adj_config.output_subdir) + os.makedirs(adj_dir, exist_ok=True) + return adj_dir + + def _save_decisions(self) -> None: + """Persist all decisions to disk.""" + try: + adj_dir = self._get_output_dir() + decisions_file = os.path.join(adj_dir, "decisions.json") + + data = { + "decisions": [d.to_dict() for d in self.decisions.values()], + "last_updated": datetime.now().isoformat(), + } + + with open(decisions_file, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + + except Exception as e: + self.logger.error(f"Failed to save adjudication decisions: {e}") + + def _load_decisions(self) -> None: + """Load previously saved decisions from disk.""" + try: + output_dir = self.config.get("output_annotation_dir", "annotation_output") + adj_dir = os.path.join(output_dir, self.adj_config.output_subdir) + decisions_file = os.path.join(adj_dir, "decisions.json") + + if not os.path.exists(decisions_file): + return + + with open(decisions_file, "r", encoding="utf-8") as f: + data = json.load(f) + + for d in data.get("decisions", []): + decision = AdjudicationDecision.from_dict(d) + self.decisions[decision.instance_id] = decision + + self.logger.info( + f"Loaded {len(self.decisions)} previous adjudication decisions" + ) + + except Exception as e: + self.logger.warning(f"Failed to load adjudication decisions: {e}") + + def generate_final_dataset(self) -> List[Dict[str, Any]]: + """ + Generate the final dataset by merging unanimous agreements + and adjudication decisions. + + Returns: + List of item dicts with final labels and provenance + """ + from potato.user_state_management import get_user_state_manager + from potato.item_state_management import get_item_state_manager + + usm = get_user_state_manager() + ism = get_item_state_manager() + + annotation_schemes = self.config.get("annotation_schemes", []) + scheme_names = [s.get("name", "") for s in annotation_schemes] + results = [] + + for instance_id, item in ism.instance_id_to_instance.items(): + instance_id_str = str(instance_id) + result = { + "instance_id": instance_id_str, + "item_data": item.get_data() if hasattr(item, 'get_data') else {}, + } + + # Check if we have an adjudication decision + decision = self.decisions.get(instance_id_str) + if decision: + result["labels"] = decision.label_decisions + result["spans"] = decision.span_decisions + result["source"] = "adjudicated" + result["adjudicator"] = decision.adjudicator_id + result["confidence"] = decision.confidence + result["provenance"] = decision.source + results.append(result) + continue + + # Check for unanimous agreement + annotators = ism.instance_annotators.get(instance_id, set()) + annotators = { + u for u in annotators + if u not in self.adj_config.adjudicator_users + } + + if len(annotators) < 2: + continue + + # Collect annotations + annotations = {} + for user_id in annotators: + user_state = usm.get_user_state(user_id) + if not user_state: + continue + labels = user_state.instance_id_to_label_to_value.get( + instance_id_str, {} + ) + if labels: + annotations[user_id] = self._serialize_labels(labels) + + if not annotations: + continue + + # Check for unanimity per schema + unanimous_labels = {} + is_unanimous = True + for schema in scheme_names: + values = [] + for user_annots in annotations.values(): + if schema in user_annots: + values.append(json.dumps(user_annots[schema], sort_keys=True)) + + if len(values) < 2: + continue + + if len(set(values)) == 1: + unanimous_labels[schema] = json.loads(values[0]) + else: + is_unanimous = False + + if is_unanimous and unanimous_labels: + result["labels"] = unanimous_labels + result["source"] = "unanimous" + result["num_annotators"] = len(annotators) + results.append(result) + else: + result["labels"] = {} + result["source"] = "unresolved" + result["num_annotators"] = len(annotators) + results.append(result) + + return results + + +def init_adjudication_manager(config: Dict[str, Any]) -> Optional[AdjudicationManager]: + """Initialize the singleton AdjudicationManager.""" + global _ADJUDICATION_MANAGER + + with _ADJUDICATION_LOCK: + if _ADJUDICATION_MANAGER is None: + _ADJUDICATION_MANAGER = AdjudicationManager(config) + + return _ADJUDICATION_MANAGER + + +def get_adjudication_manager() -> Optional[AdjudicationManager]: + """Get the singleton AdjudicationManager instance.""" + return _ADJUDICATION_MANAGER + + +def clear_adjudication_manager(): + """Clear the singleton (for testing).""" + global _ADJUDICATION_MANAGER + with _ADJUDICATION_LOCK: + _ADJUDICATION_MANAGER = None diff --git a/potato/adjudication_export.py b/potato/adjudication_export.py new file mode 100644 index 0000000000000000000000000000000000000000..d1ce83f7f9a71aae460d3da85622b31e7f7a01cf --- /dev/null +++ b/potato/adjudication_export.py @@ -0,0 +1,162 @@ +""" +Adjudication Export CLI + +Generate final datasets by merging unanimous agreements and adjudication decisions. + +Usage: + python -m potato.adjudication_export --config config.yaml --output final_dataset.jsonl + python -m potato.adjudication_export --config config.yaml --output final.csv --format csv + python -m potato.adjudication_export --config config.yaml --output final.json --format json +""" + +import argparse +import csv +import json +import os +import sys +import logging + +logger = logging.getLogger(__name__) + + +def main(): + parser = argparse.ArgumentParser( + description="Export adjudicated dataset from Potato annotation project" + ) + parser.add_argument( + "--config", required=True, + help="Path to the Potato config YAML file" + ) + parser.add_argument( + "--output", required=True, + help="Output file path" + ) + parser.add_argument( + "--format", choices=["jsonl", "json", "csv"], default="jsonl", + help="Output format (default: jsonl)" + ) + parser.add_argument( + "--include-unresolved", action="store_true", + help="Include items without adjudication or consensus" + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Verbose output" + ) + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + # Load config + from potato.server_utils.config_module import init_config, config + try: + init_config(args.config) + except Exception as e: + print(f"Error loading config: {e}", file=sys.stderr) + sys.exit(1) + + # Initialize state managers + from potato.item_state_management import init_item_state_manager + from potato.user_state_management import init_user_state_manager + + init_user_state_manager(config) + init_item_state_manager(config) + + # Load data (this loads items and user annotations from disk) + # We need a minimal load - just items and user states + from potato.flask_server import load_instance_data, load_user_data + load_instance_data(config) + load_user_data(config) + + # Initialize adjudication manager + from potato.adjudication import init_adjudication_manager + adj_mgr = init_adjudication_manager(config) + + if not adj_mgr or not adj_mgr.adj_config.enabled: + print("Adjudication is not enabled in this config.", file=sys.stderr) + sys.exit(1) + + # Build queue to compute agreements + adj_mgr.build_queue() + + # Generate final dataset + results = adj_mgr.generate_final_dataset() + + # Filter unresolved if not requested + if not args.include_unresolved: + results = [r for r in results if r.get("source") != "unresolved"] + + # Write output + output_path = args.output + fmt = args.format + + if fmt == "jsonl": + with open(output_path, "w") as f: + for item in results: + f.write(json.dumps(item) + "\n") + + elif fmt == "json": + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + + elif fmt == "csv": + if not results: + print("No results to export.", file=sys.stderr) + sys.exit(0) + + # Flatten for CSV + fieldnames = set() + flat_results = [] + for item in results: + flat = { + "instance_id": item["instance_id"], + "source": item.get("source", ""), + } + # Flatten labels + labels = item.get("labels", {}) + for schema, value in labels.items(): + if isinstance(value, dict): + flat[schema] = json.dumps(value) + else: + flat[schema] = value + + # Add provenance fields + if "adjudicator" in item: + flat["adjudicator"] = item["adjudicator"] + if "confidence" in item: + flat["confidence"] = item["confidence"] + if "num_annotators" in item: + flat["num_annotators"] = item["num_annotators"] + + fieldnames.update(flat.keys()) + flat_results.append(flat) + + # Sort fieldnames for consistent output + fieldnames = sorted(fieldnames) + + with open(output_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(flat_results) + + # Summary + total = len(results) + unanimous = sum(1 for r in results if r.get("source") == "unanimous") + adjudicated = sum(1 for r in results if r.get("source") == "adjudicated") + unresolved = sum(1 for r in results if r.get("source") == "unresolved") + + print(f"\nExport complete: {output_path}") + print(f" Total items: {total}") + print(f" Unanimous: {unanimous}") + print(f" Adjudicated: {adjudicated}") + if args.include_unresolved: + print(f" Unresolved: {unresolved}") + print(f" Format: {fmt}") + + +if __name__ == "__main__": + main() diff --git a/potato/admin.py b/potato/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..3ebc8733ac29b6c54f8546db1b24ccf45c7579d2 --- /dev/null +++ b/potato/admin.py @@ -0,0 +1,2253 @@ +""" +Admin Dashboard Module + +This module provides comprehensive admin functionality for the annotation platform, +including dashboard data generation, timing analysis, and configuration management. + +The admin dashboard offers: +- Real-time overview of annotation progress and statistics +- Detailed annotator performance metrics and timing analysis +- Instance-level annotation tracking and disagreement analysis +- Configuration management and system state monitoring +- Question and annotation scheme analysis +- User progress tracking and completion statistics +- Comprehensive annotation history tracking and suspicious activity detection +- Performance metrics and quality assurance monitoring +- Session tracking and behavioral analysis + +Key Components: +- AdminDashboard: Main class for admin functionality +- AnnotatorTimingData: Data class for annotator timing information +- InstanceData: Data class for instance information and statistics +- Dashboard data generation and analysis functions +- Configuration update and management functions +- AnnotationHistoryAnalyzer: Advanced history analysis and suspicious activity detection + +The dashboard provides insights into: +- Overall annotation progress and completion rates +- Individual annotator performance and efficiency +- Annotation quality through disagreement analysis +- System configuration and operational status +- Real-time monitoring of active annotation sessions +- Fine-grained annotation timing and behavioral patterns +- Suspicious activity detection and quality assurance +- Session-based performance analysis + +Access Control: +- Admin access is controlled via API key authentication +- Debug mode allows admin access without API key +- All admin endpoints require proper authentication +""" + +import json +import logging +import datetime +from typing import Dict, List, Optional, Tuple, Any +from collections import defaultdict, Counter +from dataclasses import dataclass +from flask import request, jsonify, session + +from potato.flask_server import ( + config, logger, get_user_state_manager, get_item_state_manager, + get_users, get_total_annotations +) +from potato.annotation_history import AnnotationHistoryManager, AnnotationAction +from potato.quality_control import get_quality_control_manager + +@dataclass +class AnnotatorTimingData: + """ + Data class for annotator timing information. + + This class encapsulates timing metrics for individual annotators, + including total annotations, working time, and performance statistics. + Now enhanced with annotation history tracking and suspicious activity detection. + """ + user_id: str + total_annotations: int + total_seconds: int + average_seconds_per_annotation: float + last_activity: Optional[datetime.datetime] + current_instance_time: Optional[int] + annotations_per_hour: float + phase: str + has_assignments: bool + remaining_assignments: bool + + # Annotation history metrics + total_actions: int + average_action_time_ms: float + fastest_action_time_ms: int + slowest_action_time_ms: int + actions_per_minute: float + suspicious_score: float + suspicious_level: str + fast_actions_count: int + burst_actions_count: int + session_start_time: Optional[datetime.datetime] + current_session_duration_minutes: Optional[float] + recent_actions_count: int # Actions in last 5 minutes + + # Training metrics + training_completed: bool + training_correct_answers: int + training_total_attempts: int + training_pass_rate: float + training_current_question: int + training_total_questions: int + +@dataclass +class InstanceData: + """ + Data class for instance information. + + This class encapsulates information about annotation instances, + including annotation counts, disagreement scores, and annotator lists. + """ + id: str + text: str + displayed_text: str + annotation_count: int + completion_percentage: float + most_frequent_label: Optional[str] + label_disagreement: float + annotators: List[str] + num_ai_instance: int + average_time_per_annotation: Optional[float] + +class AdminDashboard: + """ + Main class for admin dashboard functionality. + + This class provides comprehensive admin features including dashboard + data generation, timing analysis, configuration management, and + system monitoring capabilities. + """ + + def __init__(self): + """Initialize the admin dashboard.""" + self.logger = logging.getLogger(__name__) + + def check_admin_access(self) -> bool: + """ + Check if the current request has admin access via API key. + + Validates against all key sources (config, env var, auto-generated file) + and accepts keys from X-API-Key header or session. + + Returns: + bool: True if admin access is granted, False otherwise + """ + from potato.server_utils.admin_key import validate_admin_api_key + api_key = request.headers.get('X-API-Key') or session.get('admin_api_key') + return validate_admin_api_key(api_key, config) + + def get_dashboard_overview(self) -> Dict[str, Any]: + """ + Get comprehensive dashboard overview data. + + This method generates a complete overview of the annotation system, + including user statistics, annotation progress, and system configuration. + + Returns: + Dict containing overview statistics with the following structure: + - overview: User counts, annotation counts, completion percentages + - config: System configuration and settings + + Side Effects: + - Logs errors if data generation fails + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + usm = get_user_state_manager() + ism = get_item_state_manager() + + # Get all users and their states + users = get_users() + total_annotations = get_total_annotations() + + # Calculate user statistics + active_users = 0 + completed_users = 0 + total_working_time = 0 + + for username in users: + user_state = usm.get_user_state(username) + if user_state: + if user_state.get_phase().value == "ANNOTATION": + active_users += 1 + elif user_state.get_phase().value == "DONE": + completed_users += 1 + + # Get timing data + timing_data = self._get_annotator_timing_data(username) + if timing_data: + total_working_time += timing_data.total_seconds + + # Get item statistics + items = ism.items() + items_with_annotations = 0 + total_assignments = 0 + + for item in items: + item_id = item.get_id() + annotators = ism.get_annotators_for_item(item_id) + if annotators: + items_with_annotations += 1 + total_assignments += len(annotators) + + # Calculate completion percentages + total_items = len(items) + completion_percentage = (items_with_annotations / total_items * 100) if total_items > 0 else 0 + + # Format total working time + hours = total_working_time // 3600 + minutes = (total_working_time % 3600) // 60 + formatted_time = f"{hours}h {minutes}m" + + return { + "overview": { + "total_users": len(users), + "active_users": active_users, + "completed_users": completed_users, + "total_annotations": total_annotations, + "total_items": total_items, + "items_with_annotations": items_with_annotations, + "completion_percentage": round(completion_percentage, 1), + "total_assignments": total_assignments, + "total_working_time": formatted_time, + "average_annotations_per_item": round(total_annotations / total_items, 1) if total_items > 0 else 0 + }, + "config": { + "annotation_task_name": config.get("annotation_task_name", "Unknown"), + "max_annotations_per_user": config.get("max_annotations_per_user", "Unlimited"), + "max_annotations_per_item": config.get("max_annotations_per_item", "Unlimited"), + "assignment_strategy": config.get("assignment_strategy", "fixed_order"), + "debug_mode": config.get("debug", False) + } + } + + except Exception as e: + self.logger.error(f"Error getting dashboard overview: {e}") + return {"error": f"Failed to get dashboard overview: {str(e)}"}, 500 + + def get_annotators_data(self) -> Dict[str, Any]: + """ + Get detailed annotator data including timing information. + + Returns: + Dict containing annotator data with timing analysis + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + usm = get_user_state_manager() + users = get_users() + annotators_data = [] + + + for username in users: + user_state = usm.get_user_state(username) + if user_state: + timing_data = self._get_annotator_timing_data(username) + if timing_data: + annotators_data.append({ + "user_id": timing_data.user_id, + "total_annotations": timing_data.total_annotations, + "completion_percentage": self._calculate_completion_percentage(timing_data.user_id), + "total_seconds": timing_data.total_seconds, + "average_seconds_per_annotation": timing_data.average_seconds_per_annotation, + "annotations_per_hour": timing_data.annotations_per_hour, + "phase": timing_data.phase, + "has_assignments": timing_data.has_assignments, + "remaining_assignments": timing_data.remaining_assignments, + "max_assignments": user_state.get_max_assignments(), + "last_activity": timing_data.last_activity.isoformat() if timing_data.last_activity else None, + "current_instance_time": timing_data.current_instance_time, + + # NEW: Annotation history metrics + "total_actions": timing_data.total_actions, + "average_action_time_ms": timing_data.average_action_time_ms, + "fastest_action_time_ms": timing_data.fastest_action_time_ms if timing_data.fastest_action_time_ms != float('inf') else None, + "slowest_action_time_ms": timing_data.slowest_action_time_ms, + "actions_per_minute": timing_data.actions_per_minute, + "suspicious_score": timing_data.suspicious_score, + "suspicious_level": timing_data.suspicious_level, + "fast_actions_count": timing_data.fast_actions_count, + "burst_actions_count": timing_data.burst_actions_count, + "session_start_time": timing_data.session_start_time.isoformat() if timing_data.session_start_time else None, + "current_session_duration_minutes": timing_data.current_session_duration_minutes, + "recent_actions_count": timing_data.recent_actions_count, + + # Training metrics + "training_completed": timing_data.training_completed, + "training_correct_answers": timing_data.training_correct_answers, + "training_total_attempts": timing_data.training_total_attempts, + "training_pass_rate": round(timing_data.training_pass_rate, 2), + "training_current_question": timing_data.training_current_question, + "training_total_questions": timing_data.training_total_questions + }) + + # Sort by suspicious score (highest first) + annotators_data.sort(key=lambda x: x["suspicious_score"], reverse=True) + + return { + "total_annotators": len(annotators_data), + "annotators": annotators_data, + "summary": { + "high_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] in ["High", "Very High"]]), + "medium_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] == "Medium"]), + "low_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] == "Low"]), + "normal_count": len([a for a in annotators_data if a["suspicious_level"] == "Normal"]), + "average_suspicious_score": sum(a["suspicious_score"] for a in annotators_data) / len(annotators_data) if annotators_data else 0 + } + } + + except Exception as e: + self.logger.error(f"Error getting annotators data: {e}") + return {"error": f"Failed to get annotators data: {str(e)}"}, 500 + + def get_annotation_history_data(self, user_id: Optional[str] = None, + instance_id: Optional[str] = None, + minutes: Optional[int] = None) -> Dict[str, Any]: + """ + Get detailed annotation history data with filtering options. + + Args: + user_id: Optional user ID to filter by + instance_id: Optional instance ID to filter by + minutes: Optional time window in minutes + + Returns: + Dict containing annotation history data + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + usm = get_user_state_manager() + + if user_id: + # Get history for specific user + user_state = usm.get_user_state(user_id) + if not user_state: + return {"error": f"User {user_id} not found"}, 404 + + actions = user_state.get_annotation_history(instance_id) + if minutes: + actions = user_state.get_recent_actions(minutes) + + return self._format_annotation_history(actions, user_id) + else: + # Get history for all users + all_actions = [] + users = get_users() + + for username in users: + user_state = usm.get_user_state(username) + if user_state: + user_actions = user_state.get_annotation_history(instance_id) + if minutes: + user_actions = user_state.get_recent_actions(minutes) + all_actions.extend(user_actions) + + return self._format_annotation_history(all_actions, "all_users") + + except Exception as e: + self.logger.error(f"Error getting annotation history data: {e}") + return {"error": f"Failed to get annotation history data: {str(e)}"}, 500 + + def get_suspicious_activity_data(self) -> Dict[str, Any]: + """ + Get comprehensive suspicious activity analysis. + + Returns: + Dict containing suspicious activity data + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + usm = get_user_state_manager() + users = get_users() + suspicious_data = [] + + for username in users: + user_state = usm.get_user_state(username) + if user_state: + suspicious_actions = user_state.get_suspicious_activity() + if suspicious_actions: + suspicious_data.append({ + "user_id": username, + "suspicious_actions_count": len(suspicious_actions), + "suspicious_actions": [ + { + "action_id": action.action_id, + "timestamp": action.timestamp.isoformat(), + "instance_id": action.instance_id, + "action_type": action.action_type, + "schema_name": action.schema_name, + "label_name": action.label_name, + "server_processing_time_ms": action.server_processing_time_ms, + "session_id": action.session_id + } + for action in suspicious_actions[:10] # Limit to 10 most recent + ] + }) + + return { + "total_users_with_suspicious_activity": len(suspicious_data), + "suspicious_activity": suspicious_data + } + + except Exception as e: + self.logger.error(f"Error getting suspicious activity data: {e}") + return {"error": f"Failed to get suspicious activity data: {str(e)}"}, 500 + + def get_instances_data(self, page: int = 1, page_size: int = 25, + sort_by: str = "annotation_count", sort_order: str = "desc", + filter_completion: Optional[str] = None) -> Dict[str, Any]: + """ + Get paginated instances data with sorting and filtering. + + Args: + page: Page number (1-based) + page_size: Number of instances per page + sort_by: Field to sort by (annotation_count, completion_percentage, disagreement, id) + sort_order: Sort order (asc, desc) + filter_completion: Filter by completion status (completed, incomplete, all) + + Returns: + Dict containing paginated instances data + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + ism = get_item_state_manager() + items = ism.items() + + # Convert items to InstanceData objects + instances_data = [] + for item in items: + item_id = item.get_id() + annotators = ism.get_annotators_for_item(item_id) + annotation_count = len(annotators) if annotators else 0 + + # Calculate completion percentage + max_annotations = config.get("max_annotations_per_item", -1) + if max_annotations > 0: + completion_percentage = min(100, (annotation_count / max_annotations) * 100) + else: + completion_percentage = 100 if annotation_count > 0 else 0 + + # Calculate most frequent label and disagreement + most_frequent_label, disagreement = self._calculate_label_statistics(item_id) + + # Calculate average time per annotation + avg_time = self._calculate_average_time_per_annotation(item_id) + + instance_data = InstanceData( + id=item_id, + text=item.get_text(), + displayed_text=item.get_displayed_text(), + annotation_count=annotation_count, + completion_percentage=completion_percentage, + most_frequent_label=most_frequent_label, + label_disagreement=disagreement, + annotators=list(annotators) if annotators else [], + average_time_per_annotation=avg_time, + num_ai_instance=self._calculate_total_instance_ai(item_id) + ) + instances_data.append(instance_data) + + # Apply filters + if filter_completion == "completed": + instances_data = [i for i in instances_data if i.completion_percentage >= 100] + elif filter_completion == "incomplete": + instances_data = [i for i in instances_data if i.completion_percentage < 100] + + # Apply sorting + reverse = sort_order.lower() == "desc" + if sort_by == "annotation_count": + instances_data.sort(key=lambda x: x.annotation_count, reverse=reverse) + elif sort_by == "completion_percentage": + instances_data.sort(key=lambda x: x.completion_percentage, reverse=reverse) + elif sort_by == "disagreement": + instances_data.sort(key=lambda x: x.label_disagreement, reverse=reverse) + elif sort_by == "id": + instances_data.sort(key=lambda x: x.id, reverse=reverse) + elif sort_by == "average_time": + instances_data.sort(key=lambda x: x.average_time_per_annotation or 0, reverse=reverse) + + # Apply pagination + total_instances = len(instances_data) + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + paginated_instances = instances_data[start_idx:end_idx] + + # Convert to serializable format + serialized_instances = [] + for instance in paginated_instances: + serialized_instances.append({ + "id": instance.id, + "text": instance.text[:100] + "..." if len(instance.text) > 100 else instance.text, + "displayed_text": instance.displayed_text[:100] + "..." if len(instance.displayed_text) > 100 else instance.displayed_text, + "annotation_count": instance.annotation_count, + "completion_percentage": round(instance.completion_percentage, 1), + "most_frequent_label": instance.most_frequent_label, + "label_disagreement": round(instance.label_disagreement, 2), + "annotators": instance.annotators, + "num_ai_instance": instance.num_ai_instance, + "average_time_per_annotation": self._format_seconds(instance.average_time_per_annotation) if instance.average_time_per_annotation else None + }) + + return { + "instances": serialized_instances, + "pagination": { + "page": page, + "page_size": page_size, + "total_instances": total_instances, + "total_pages": (total_instances + page_size - 1) // page_size, + "has_next": end_idx < total_instances, + "has_prev": page > 1 + }, + "summary": { + "completed_instances": len([i for i in instances_data if i.completion_percentage >= 100]), + "incomplete_instances": len([i for i in instances_data if i.completion_percentage < 100]), + "average_annotations_per_instance": round(sum(i.annotation_count for i in instances_data) / len(instances_data), 1) if instances_data else 0, + "average_disagreement": round(sum(i.label_disagreement for i in instances_data) / len(instances_data), 2) if instances_data else 0 + } + } + + except Exception as e: + self.logger.error(f"Error getting instances data: {e}") + return {"error": f"Failed to get instances data: {str(e)}"}, 500 + + def update_config(self, config_updates: Dict[str, Any]) -> Dict[str, Any]: + """ + Update system configuration. + + Args: + config_updates: Dictionary of configuration updates + + Returns: + Dict containing update result + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + # Validate and apply updates + updated_fields = [] + + for key, value in config_updates.items(): + if key in ["max_annotations_per_user", "max_annotations_per_item"]: + if isinstance(value, int) and value >= -1: + config[key] = value + updated_fields.append(key) + else: + return {"error": f"Invalid value for {key}: must be integer >= -1"}, 400 + + elif key == "assignment_strategy": + valid_strategies = ["random", "fixed_order", "least_annotated", "max_diversity", "active_learning", "llm_confidence"] + if value in valid_strategies: + config[key] = value + updated_fields.append(key) + else: + return {"error": f"Invalid assignment strategy: {value}"}, 400 + + return { + "status": "success", + "message": f"Updated configuration fields: {', '.join(updated_fields)}", + "updated_fields": updated_fields + } + + except Exception as e: + self.logger.error(f"Error updating config: {e}") + return {"error": f"Failed to update config: {str(e)}"}, 500 + + def get_questions_data(self) -> Dict[str, Any]: + """ + Get aggregate analysis data for each annotation schema/question. + + Returns: + Dict containing questions data with visualizations for different annotation types + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + ism = get_item_state_manager() + annotation_schemes = config.get("annotation_schemes", []) + questions_data = [] + + users = get_users() + + for scheme in annotation_schemes: + scheme_name = scheme.get("name", "Unknown") + annotation_type = scheme.get("annotation_type", "unknown") + + all_annotations = [] + item_annotations = {} + + for item in ism.items(): + item_id = item.get_id() + item_annotations[item_id] = [] + + for username in users: + user_state = get_user_state_manager().get_user_state(username) + if user_state: + label_annotations = user_state.get_label_annotations(item_id) + for label, value in label_annotations.items(): + label_schema = None + label_name = None + if hasattr(label, 'get_schema'): + label_schema = label.get_schema() + label_name = label.get_name() + elif hasattr(label, 'schema'): + label_schema = label.schema + label_name = getattr(label, 'name', None) + elif isinstance(label, str): + label_schema = label + + if label_schema == scheme_name: + normalized_value = label_name if label_name else value + + if annotation_type in ["radio", "select"]: + normalized_value = self._normalize_categorical_value(normalized_value) + elif annotation_type == "multiselect" and isinstance(normalized_value, list): + normalized_value = [ + normalized_label + for normalized_label in ( + self._normalize_categorical_value(v) for v in normalized_value + ) + if normalized_label is not None + ] + + if normalized_value is not None: + all_annotations.append(normalized_value) + item_annotations[item_id].append(normalized_value) + + analysis = self._analyze_annotation_scheme( + annotation_type, scheme, all_annotations, item_annotations + ) + + questions_data.append({ + "name": scheme_name, + "type": annotation_type, + "description": scheme.get("description", ""), + "total_annotations": len(all_annotations), + "items_with_annotations": len([item_id for item_id, annotations in item_annotations.items() if annotations]), + "analysis": analysis + }) + + return { + "questions": questions_data, + "summary": { + "total_questions": len(questions_data), + "total_annotations": sum(q["total_annotations"] for q in questions_data), + "question_types": list(set(q["type"] for q in questions_data)) + } + } + + except Exception as e: + self.logger.error(f"Error getting questions data: {e}") + return {"error": f"Failed to get questions data: {str(e)}"}, 500 + + def _analyze_annotation_scheme(self, annotation_type: str, scheme: dict, + all_annotations: list, item_annotations: dict) -> dict: + """ + Analyze annotations based on their type and generate appropriate visualizations. + """ + if not all_annotations: + return {"error": "No annotations found"} + + analysis = { + "type": annotation_type, + "total_count": len(all_annotations) + } + + if annotation_type in ["radio", "select"]: + normalized_annotations = [ + normalized for normalized in + (self._normalize_categorical_value(annotation) for annotation in all_annotations) + if normalized is not None + ] + if not normalized_annotations: + return {"error": "No annotations found"} + + label_counts = Counter(normalized_annotations) + raw_labels = scheme.get("labels", []) + labels = [ + normalized for normalized in + (self._normalize_categorical_value(label) for label in raw_labels) + if normalized is not None + ] + + analysis.update({ + "visualization_type": "histogram", + "data": { + "labels": labels, + "counts": [label_counts.get(label, 0) for label in labels], + "percentages": [round(label_counts.get(label, 0) / len(normalized_annotations) * 100, 1) + for label in labels] + }, + "most_common": label_counts.most_common(1)[0] if label_counts else None, + "agreement_score": self._calculate_agreement_score(item_annotations) + }) + elif annotation_type == "multiselect": + # Multi-label data - show label frequency and co-occurrence + label_counts = Counter() + co_occurrence = defaultdict(int) + labels = scheme.get("labels", []) + + for annotations in item_annotations.values(): + if isinstance(annotations, list): + # Count individual labels + for annotation in annotations: + if isinstance(annotation, list): + for label in annotation: + label_counts[label] += 1 + + # Count co-occurrences + for i, annotation1 in enumerate(annotations): + if isinstance(annotation1, list): + for j, annotation2 in enumerate(annotations): + if i != j and isinstance(annotation2, list): + for label1 in annotation1: + for label2 in annotation2: + if label1 < label2: + co_occurrence[(label1, label2)] += 1 + + analysis.update({ + "visualization_type": "multiselect_analysis", + "data": { + "labels": labels, + "counts": [label_counts.get(label, 0) for label in labels], + "percentages": [round(label_counts.get(label, 0) / len(item_annotations) * 100, 1) + for label in labels], + "co_occurrence": dict(co_occurrence) + }, + "most_common": label_counts.most_common(3) if label_counts else [], + "average_labels_per_item": round(sum(len(ann) if isinstance(ann, list) else 1 + for anns in item_annotations.values() + for ann in anns) / len(all_annotations), 2) + }) + + elif annotation_type in ["likert", "number", "slider"]: + # Numeric data - show distribution and statistics + numeric_values = [] + for value in all_annotations: + try: + if isinstance(value, (int, float)): + numeric_values.append(float(value)) + elif isinstance(value, str) and value.replace('.', '').replace('-', '').isdigit(): + numeric_values.append(float(value)) + except (ValueError, TypeError): + continue + + if numeric_values: + analysis.update({ + "visualization_type": "distribution", + "data": { + "values": numeric_values, + "bins": self._create_histogram_bins(numeric_values, scheme), + "statistics": { + "mean": round(sum(numeric_values) / len(numeric_values), 2), + "median": round(sorted(numeric_values)[len(numeric_values)//2], 2), + "min": min(numeric_values), + "max": max(numeric_values), + "std": round((sum((x - sum(numeric_values)/len(numeric_values))**2 + for x in numeric_values) / len(numeric_values))**0.5, 2) + } + }, + "range": scheme.get("min", 0) if "min" in scheme else None, + "max": scheme.get("max", 10) if "max" in scheme else None + }) + else: + analysis["error"] = "No valid numeric values found" + + elif annotation_type == "text": + # Text data - show length distribution and common patterns + text_lengths = [] + word_counts = [] + common_words = Counter() + + for value in all_annotations: + if isinstance(value, str) and value.strip(): + text_lengths.append(len(value)) + words = value.lower().split() + word_counts.append(len(words)) + common_words.update(words) + + if text_lengths: + analysis.update({ + "visualization_type": "text_analysis", + "data": { + "lengths": text_lengths, + "word_counts": word_counts, + "common_words": common_words.most_common(10), + "statistics": { + "avg_length": round(sum(text_lengths) / len(text_lengths), 1), + "avg_words": round(sum(word_counts) / len(word_counts), 1), + "min_length": min(text_lengths), + "max_length": max(text_lengths), + "empty_responses": len([v for v in all_annotations + if not isinstance(v, str) or not v.strip()]) + } + } + }) + else: + analysis["error"] = "No valid text responses found" + + elif annotation_type == "span": + # Span data - show coverage and overlap statistics + span_counts = [] + total_spans = 0 + + for annotations in item_annotations.values(): + if isinstance(annotations, list): + for annotation in annotations: + if isinstance(annotation, list): + span_counts.append(len(annotation)) + total_spans += len(annotation) + + if span_counts: + analysis.update({ + "visualization_type": "span_analysis", + "data": { + "span_counts": span_counts, + "total_spans": total_spans, + "statistics": { + "avg_spans_per_item": round(sum(span_counts) / len(span_counts), 2), + "items_with_spans": len([c for c in span_counts if c > 0]), + "max_spans": max(span_counts) if span_counts else 0, + "min_spans": min(span_counts) if span_counts else 0 + } + } + }) + else: + analysis["error"] = "No valid span annotations found" + + else: + analysis["error"] = f"Unsupported annotation type: {annotation_type}" + + return analysis + + def _calculate_agreement_score(self, item_annotations: dict) -> float: + """Calculate agreement score for categorical annotations.""" + if not item_annotations: + return 0.0 + + agreement_scores = [] + for annotations in item_annotations.values(): + if len(annotations) > 1: + # Calculate percentage of most common annotation + counter = Counter(annotations) + most_common_count = counter.most_common(1)[0][1] + agreement_scores.append(most_common_count / len(annotations)) + + return round(sum(agreement_scores) / len(agreement_scores) * 100, 1) if agreement_scores else 0.0 + + def _create_histogram_bins(self, values: list, scheme: dict) -> dict: + """Create histogram bins for numeric data.""" + if not values: + return {"bins": [], "counts": []} + + min_val = scheme.get("min", min(values)) + max_val = scheme.get("max", max(values)) + + # Create 10 bins + bin_size = (max_val - min_val) / 10 + bins = [min_val + i * bin_size for i in range(11)] + counts = [0] * 10 + + for value in values: + bin_index = min(int((value - min_val) / bin_size), 9) + counts[bin_index] += 1 + + return { + "bins": [round(b, 2) for b in bins], + "counts": counts + } + + def _get_annotator_timing_data(self, user_id: str) -> Optional[AnnotatorTimingData]: + """ + Get timing data for a specific annotator. + + Args: + user_id: The user ID to get timing data for + + Returns: + AnnotatorTimingData object or None if user not found + """ + try: + usm = get_user_state_manager() + user_state = usm.get_user_state(user_id) + + if not user_state: + return None + + # Get basic user info + total_annotations = len(user_state.get_all_annotations()) + phase = str(user_state.get_phase()) + has_assignments = user_state.has_assignments() + remaining_assignments = user_state.has_remaining_assignments() + + # Calculate timing data + total_seconds = 0 + instance_times = [] + + for instance_id, behavioral_data in user_state.instance_id_to_behavioral_data.items(): + instance_seconds = None + # Handle both BehavioralData objects and plain dicts + if hasattr(behavioral_data, 'total_time_ms'): + # BehavioralData object (loaded from JSON) + if behavioral_data.total_time_ms: + instance_seconds = behavioral_data.total_time_ms / 1000.0 + elif isinstance(behavioral_data, dict): + # Plain dict (runtime data) + if behavioral_data.get("total_time_ms"): + instance_seconds = behavioral_data["total_time_ms"] / 1000.0 + elif behavioral_data.get("time_string"): + parsed_time = user_state.parse_time_string(behavioral_data["time_string"]) + if parsed_time: + instance_seconds = parsed_time["total_seconds"] + if instance_seconds is not None: + total_seconds += instance_seconds + instance_times.append(instance_seconds) + + # Calculate averages + average_seconds_per_annotation = total_seconds / total_annotations if total_annotations > 0 else 0 + annotations_per_hour = (total_annotations * 3600) / total_seconds if total_seconds > 0 else 0 + + # Get current instance time (if any) + current_instance_time = None + current_instance = user_state.get_current_instance() + if current_instance: + current_instance_id = current_instance.get_id() + current_behavioral = user_state.instance_id_to_behavioral_data.get(current_instance_id) + if current_behavioral: + if hasattr(current_behavioral, 'total_time_ms'): + if current_behavioral.total_time_ms: + current_instance_time = current_behavioral.total_time_ms / 1000.0 + elif isinstance(current_behavioral, dict): + if current_behavioral.get("total_time_ms"): + current_instance_time = current_behavioral["total_time_ms"] / 1000.0 + elif current_behavioral.get("time_string"): + parsed_current = user_state.parse_time_string(current_behavioral["time_string"]) + if parsed_current: + current_instance_time = parsed_current["total_seconds"] + + # Estimate last activity (for now, use current time - this could be enhanced) + last_activity = datetime.datetime.now() + + # NEW: Get annotation history metrics + performance_metrics = user_state.get_performance_metrics() + suspicious_analysis = AnnotationHistoryManager.detect_suspicious_activity( + user_state.get_annotation_history() + ) + recent_actions = user_state.get_recent_actions(5) # Last 5 minutes + + # Calculate session duration + current_session_duration_minutes = None + if user_state.session_start_time: + duration = datetime.datetime.now() - user_state.session_start_time + current_session_duration_minutes = duration.total_seconds() / 60 + + # Get training statistics + training_state = user_state.get_training_state() + training_completed = training_state.is_passed() if training_state else False + training_correct_answers = training_state.get_correct_answer_count() if training_state else 0 + training_total_attempts = training_state.get_total_attempts() if training_state else 0 + training_pass_rate = (training_correct_answers / training_total_attempts * 100) if training_total_attempts > 0 else 0 + training_current_question = training_state.get_current_question_index() if training_state else 0 + training_total_questions = len(training_state.get_training_instances()) if training_state else 0 + + return AnnotatorTimingData( + user_id=user_id, + total_annotations=total_annotations, + total_seconds=total_seconds, + average_seconds_per_annotation=average_seconds_per_annotation, + last_activity=last_activity, + current_instance_time=current_instance_time, + annotations_per_hour=annotations_per_hour, + phase=phase, + has_assignments=has_assignments, + remaining_assignments=remaining_assignments, + + # NEW: Annotation history metrics + total_actions=performance_metrics.get('total_actions', 0), + average_action_time_ms=performance_metrics.get('average_action_time_ms', 0.0), + fastest_action_time_ms=performance_metrics.get('fastest_action_time_ms', 0), + slowest_action_time_ms=performance_metrics.get('slowest_action_time_ms', 0), + actions_per_minute=performance_metrics.get('actions_per_minute', 0.0), + suspicious_score=suspicious_analysis.get('suspicious_score', 0.0), + suspicious_level=suspicious_analysis.get('suspicious_level', 'Normal'), + fast_actions_count=suspicious_analysis.get('fast_actions_count', 0), + burst_actions_count=suspicious_analysis.get('burst_actions_count', 0), + session_start_time=user_state.session_start_time, + current_session_duration_minutes=current_session_duration_minutes, + recent_actions_count=len(recent_actions), + + # Training metrics + training_completed=training_completed, + training_correct_answers=training_correct_answers, + training_total_attempts=training_total_attempts, + training_pass_rate=training_pass_rate, + training_current_question=training_current_question, + training_total_questions=training_total_questions + ) + + except Exception as e: + self.logger.error(f"Error getting timing data for user {user_id}: {e}") + return None + + def _extract_behavioral_total_seconds(self, behavioral_data: Any, user_state=None) -> Optional[float]: + """Extract total annotation time in seconds from behavioral data objects or legacy dicts.""" + if not behavioral_data: + return None + + if hasattr(behavioral_data, 'total_time_ms') and behavioral_data.total_time_ms is not None: + return behavioral_data.total_time_ms / 1000.0 + + if isinstance(behavioral_data, dict): + total_time_ms = behavioral_data.get("total_time_ms") + if total_time_ms is not None: + return total_time_ms / 1000.0 + + time_string = behavioral_data.get("time_string") + if time_string and user_state and hasattr(user_state, 'parse_time_string'): + parsed_time = user_state.parse_time_string(time_string) + if parsed_time: + return parsed_time.get("total_seconds") + + return None + + def _extract_behavioral_ai_count(self, behavioral_data: Any) -> int: + """Extract AI usage count from behavioral data objects or legacy dicts.""" + if not behavioral_data: + return 0 + + if hasattr(behavioral_data, 'ai_usage'): + ai_usage = behavioral_data.ai_usage or [] + return len(ai_usage) + + if isinstance(behavioral_data, dict): + ai_usage = behavioral_data.get("ai_usage", []) or [] + return len(ai_usage) + + return 0 + + def _calculate_total_instance_ai(self, instance_id: str) -> int: + """ + Calculate total AI assistance events for an instance across all users. + + Args: + instance_id: The instance ID to analyze + + Returns: + Total number of AI usage events recorded for the instance + """ + try: + usm = get_user_state_manager() + users = get_users() + + total_ai = 0 + for username in users: + user_state = usm.get_user_state(username) + if not user_state: + continue + + behavioral_data = user_state.instance_id_to_behavioral_data.get(instance_id) + total_ai += self._extract_behavioral_ai_count(behavioral_data) + + return total_ai + + except Exception as e: + self.logger.error(f"Error calculating AI statistics for instance {instance_id}: {e}") + return 0 + + def _calculate_average_time_per_annotation(self, instance_id: str) -> Optional[float]: + """ + Calculate average time per annotation for an instance. + + Args: + instance_id: The instance ID to analyze + + Returns: + Average time in seconds or None if no data + """ + try: + usm = get_user_state_manager() + users = get_users() + + total_time = 0 + annotation_count = 0 + + for username in users: + user_state = usm.get_user_state(username) + if user_state: + behavioral_data = user_state.instance_id_to_behavioral_data.get(instance_id) + total_seconds = self._extract_behavioral_total_seconds(behavioral_data, user_state) + if total_seconds is not None: + total_time += total_seconds + annotation_count += 1 + + return total_time / annotation_count if annotation_count > 0 else None + + except Exception as e: + self.logger.error(f"Error calculating average time for instance {instance_id}: {e}") + return None + + def _calculate_completion_percentage(self, user_id: str) -> float: + """ + Calculate completion percentage for a user. + + Args: + user_id: The user ID to calculate completion for + + Returns: + Completion percentage (0-100) + """ + try: + usm = get_user_state_manager() + user_state = usm.get_user_state(user_id) + + if not user_state: + return 0.0 + + total_assignments = user_state.get_assigned_instance_count() + completed_assignments = len(user_state.get_all_annotations()) + + if total_assignments == 0: + return 0.0 + + return (completed_assignments / total_assignments) * 100 + + except Exception as e: + self.logger.error(f"Error calculating completion percentage for user {user_id}: {e}") + return 0.0 + + def _format_seconds(self, seconds: Optional[float]) -> Optional[str]: + """ + Format seconds into a human-readable string. + + Args: + seconds: Number of seconds to format + + Returns: + Formatted time string or None if input is None + """ + if seconds is None: + return None + + if seconds < 60: + return f"{int(seconds)}s" + elif seconds < 3600: + minutes = int(seconds // 60) + remaining_seconds = int(seconds % 60) + return f"{minutes}m {remaining_seconds}s" + else: + hours = int(seconds // 3600) + remaining_minutes = int((seconds % 3600) // 60) + return f"{hours}h {remaining_minutes}m" + + def _format_annotation_history(self, actions: List[AnnotationAction], context: str) -> Dict[str, Any]: + """ + Format annotation history data for API response. + + Args: + actions: List of annotation actions + context: Context string (user_id or "all_users") + + Returns: + Formatted annotation history data + """ + if not actions: + return { + "context": context, + "total_actions": 0, + "actions": [], + "summary": { + "action_types": {}, + "time_distribution": {}, + "performance_metrics": {} + } + } + + # Calculate summary statistics + action_types = Counter(action.action_type for action in actions) + time_distribution = self._calculate_time_distribution(actions) + performance_metrics = AnnotationHistoryManager.calculate_performance_metrics(actions) + + # Format actions for response + formatted_actions = [] + for action in actions[-100:]: # Limit to 100 most recent + formatted_actions.append({ + "action_id": action.action_id, + "timestamp": action.timestamp.isoformat(), + "user_id": action.user_id, + "instance_id": action.instance_id, + "action_type": action.action_type, + "schema_name": action.schema_name, + "label_name": action.label_name, + "old_value": action.old_value, + "new_value": action.new_value, + "span_data": action.span_data, + "session_id": action.session_id, + "client_timestamp": action.client_timestamp.isoformat() if action.client_timestamp else None, + "server_processing_time_ms": action.server_processing_time_ms, + "metadata": action.metadata + }) + + return { + "context": context, + "total_actions": len(actions), + "actions": formatted_actions, + "summary": { + "action_types": dict(action_types), + "time_distribution": time_distribution, + "performance_metrics": performance_metrics + } + } + + def _calculate_time_distribution(self, actions: List[AnnotationAction]) -> Dict[str, int]: + """ + Calculate time distribution of actions. + + Args: + actions: List of annotation actions + + Returns: + Dictionary with time distribution data + """ + if not actions: + return {} + + # Group by hour of day + hour_distribution = defaultdict(int) + for action in actions: + hour = action.timestamp.hour + hour_distribution[f"{hour:02d}:00"] += 1 + + return dict(hour_distribution) + + def get_crowdsourcing_data(self) -> Dict[str, Any]: + """ + Get crowdsourcing platform statistics (MTurk, Prolific). + + This method analyzes user data to provide statistics about workers + from crowdsourcing platforms like Amazon Mechanical Turk and Prolific. + + Returns: + Dict containing crowdsourcing statistics with the following structure: + - summary: Overall counts of crowdsourcing workers + - prolific: Prolific-specific statistics + - mturk: MTurk-specific statistics + - workers: List of individual worker data + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + from potato.authentication import UserAuthenticator + + usm = get_user_state_manager() + users = get_users() + + # Initialize counters + prolific_workers = [] + mturk_workers = [] + other_workers = [] + + # Track unique study/HIT IDs + prolific_study_ids = set() + mturk_hit_ids = set() + + # Get user authenticator to access stored user data + try: + user_auth = UserAuthenticator.get_instance() + user_data_store = getattr(user_auth.auth_backend, 'user_data', {}) + except (ValueError, AttributeError): + user_data_store = {} + + for username in users: + user_state = usm.get_user_state(username) + if not user_state: + continue + + # Get timing data for the user + timing_data = self._get_annotator_timing_data(username) + + # Get stored user data (from authentication) + stored_data = user_data_store.get(username, {}) + + # Determine platform based on stored data + prolific_session_id = stored_data.get('prolific_session_id') + prolific_study_id = stored_data.get('prolific_study_id') + mturk_assignment_id = stored_data.get('mturk_assignment_id') + mturk_hit_id = stored_data.get('mturk_hit_id') + + worker_info = { + "worker_id": username, + "total_annotations": timing_data.total_annotations if timing_data else 0, + "phase": timing_data.phase if timing_data else "unknown", + "total_seconds": timing_data.total_seconds if timing_data else 0, + "annotations_per_hour": timing_data.annotations_per_hour if timing_data else 0, + "completion_percentage": self._calculate_completion_percentage(username), + "suspicious_level": timing_data.suspicious_level if timing_data else "Normal", + } + + # Check for Prolific workers + if prolific_session_id or prolific_study_id or username.startswith('P'): + worker_info["platform"] = "prolific" + worker_info["session_id"] = prolific_session_id + worker_info["study_id"] = prolific_study_id + prolific_workers.append(worker_info) + if prolific_study_id: + prolific_study_ids.add(prolific_study_id) + + # Check for MTurk workers + elif mturk_assignment_id or mturk_hit_id or username.startswith('A'): + worker_info["platform"] = "mturk" + worker_info["assignment_id"] = mturk_assignment_id + worker_info["hit_id"] = mturk_hit_id + mturk_workers.append(worker_info) + if mturk_hit_id: + mturk_hit_ids.add(mturk_hit_id) + + else: + worker_info["platform"] = "other" + other_workers.append(worker_info) + + # Calculate summary statistics + all_workers = prolific_workers + mturk_workers + other_workers + + def calc_stats(workers): + if not workers: + return { + "count": 0, + "total_annotations": 0, + "total_time_seconds": 0, + "avg_annotations_per_worker": 0, + "avg_time_per_worker_minutes": 0, + "completed_count": 0, + "in_progress_count": 0, + } + total_annotations = sum(w["total_annotations"] for w in workers) + total_time = sum(w["total_seconds"] for w in workers) + completed = len([w for w in workers if w["phase"] == "Phase.DONE"]) + in_progress = len([w for w in workers if w["phase"] == "Phase.ANNOTATION"]) + return { + "count": len(workers), + "total_annotations": total_annotations, + "total_time_seconds": total_time, + "avg_annotations_per_worker": round(total_annotations / len(workers), 1) if workers else 0, + "avg_time_per_worker_minutes": round(total_time / len(workers) / 60, 1) if workers else 0, + "completed_count": completed, + "in_progress_count": in_progress, + } + + return { + "summary": { + "total_workers": len(all_workers), + "prolific_workers": len(prolific_workers), + "mturk_workers": len(mturk_workers), + "other_workers": len(other_workers), + "prolific_studies": len(prolific_study_ids), + "mturk_hits": len(mturk_hit_ids), + }, + "prolific": { + "stats": calc_stats(prolific_workers), + "study_ids": list(prolific_study_ids), + "workers": prolific_workers, + }, + "mturk": { + "stats": calc_stats(mturk_workers), + "hit_ids": list(mturk_hit_ids), + "workers": mturk_workers, + }, + "other": { + "stats": calc_stats(other_workers), + "workers": other_workers, + }, + } + + except Exception as e: + self.logger.error(f"Error getting crowdsourcing data: {e}") + return {"error": f"Failed to get crowdsourcing data: {str(e)}"}, 500 + + + def get_agreement_metrics(self) -> Dict[str, Any]: + """ + Get inter-annotator agreement metrics using Krippendorff's alpha. + + This leverages the existing agreement.py module for calculations. + + Returns: + Dict containing agreement metrics by schema and overall + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + import simpledorff + from simpledorff.metrics import nominal_metric, interval_metric + import pandas as pd + + agreement_config = config.get("agreement_metrics", {}) + min_overlap = agreement_config.get("min_overlap", 2) + + ism = get_item_state_manager() + usm = get_user_state_manager() + annotation_schemes = config.get("annotation_schemes", []) + users = get_users() + + metrics = { + "enabled": agreement_config.get("enabled", True), + "overall": {}, + "by_schema": {}, + "warnings": [] + } + + for scheme in annotation_schemes: + schema_name = scheme.get("name", "Unknown") + annotation_type = scheme.get("annotation_type", "unknown") + + # Collect annotations per item for this schema + annotations_by_item = {} + + for item in ism.items(): + item_id = item.get_id() + item_annotations = [] + + for username in users: + user_state = usm.get_user_state(username) + if not user_state: + continue + + # Get annotations for this item + all_annotations = user_state.get_all_annotations() + if item_id not in all_annotations: + continue + + instance_annotations = all_annotations[item_id] + labels = instance_annotations.get("labels", {}) + + # Find annotation for this schema + for label, value in labels.items(): + label_schema = None + if hasattr(label, 'schema'): + label_schema = label.schema + elif hasattr(label, 'get_schema'): + label_schema = label.get_schema() + + if label_schema == schema_name: + item_annotations.append({ + "user": username, + "value": value + }) + + if item_annotations: + annotations_by_item[item_id] = item_annotations + + # Filter items with minimum overlap + valid_items = { + item_id: annots + for item_id, annots in annotations_by_item.items() + if len(annots) >= min_overlap + } + + if not valid_items: + metrics["by_schema"][schema_name] = { + "error": f"No items with {min_overlap}+ annotators", + "items_count": len(annotations_by_item) + } + continue + + # Format for simpledorff + try: + reliability_data = [] + for item_id, annots in valid_items.items(): + for annot in annots: + reliability_data.append({ + "unit": item_id, + "annotator": annot["user"], + "annotation": self._normalize_annotation_value(annot["value"]) + }) + + df = pd.DataFrame(reliability_data) + + # Choose metric based on annotation type + if annotation_type in ["likert", "slider", "number"]: + metric_fn = interval_metric + metric_name = "interval" + else: + metric_fn = nominal_metric + metric_name = "nominal" + + # Calculate alpha + alpha = simpledorff.calculate_krippendorffs_alpha( + df, + experiment_col="unit", + annotator_col="annotator", + class_col="annotation", + metric_fn=metric_fn + ) + + schema_metrics = { + "krippendorff_alpha": round(alpha, 4), + "metric_type": metric_name, + "items_evaluated": len(valid_items), + "total_annotations": len(reliability_data), + "interpretation": self._interpret_alpha(alpha) + } + + # Cohen's kappa (pairwise) and Fleiss' kappa apply to + # categorical schemas; skip for interval-metric data where + # Krippendorff alpha is the appropriate measure. + if metric_name == "nominal": + try: + from potato.agreement import ( + cohen_kappa_pairwise, fleiss_kappa, + ) + schema_metrics["cohen_kappa"] = cohen_kappa_pairwise(df) + schema_metrics["fleiss_kappa"] = fleiss_kappa(df) + except Exception as e: + self.logger.error(f"Error calculating kappas for {schema_name}: {e}") + schema_metrics["kappa_error"] = str(e) + + metrics["by_schema"][schema_name] = schema_metrics + + except Exception as e: + self.logger.error(f"Error calculating alpha for {schema_name}: {e}") + metrics["by_schema"][schema_name] = { + "error": str(e), + "items_count": len(valid_items) + } + + # Calculate overall metrics + alphas = [ + m["krippendorff_alpha"] + for m in metrics["by_schema"].values() + if "krippendorff_alpha" in m + ] + if alphas: + avg_alpha = sum(alphas) / len(alphas) + metrics["overall"] = { + "average_krippendorff_alpha": round(avg_alpha, 4), + "schemas_evaluated": len(alphas), + "interpretation": self._interpret_alpha(avg_alpha) + } + + cohen_means = [ + m["cohen_kappa"]["mean_kappa"] + for m in metrics["by_schema"].values() + if isinstance(m.get("cohen_kappa"), dict) + and m["cohen_kappa"].get("mean_kappa") is not None + ] + if cohen_means: + metrics["overall"]["average_cohen_kappa"] = round( + sum(cohen_means) / len(cohen_means), 4 + ) + + fleiss_values = [ + m["fleiss_kappa"]["kappa"] + for m in metrics["by_schema"].values() + if isinstance(m.get("fleiss_kappa"), dict) + and m["fleiss_kappa"].get("kappa") is not None + ] + if fleiss_values: + metrics["overall"]["average_fleiss_kappa"] = round( + sum(fleiss_values) / len(fleiss_values), 4 + ) + + return metrics + + except ImportError as e: + self.logger.error(f"simpledorff not installed: {e}") + return { + "enabled": False, + "error": "simpledorff library not installed. Run: pip install simpledorff" + } + except Exception as e: + self.logger.error(f"Error getting agreement metrics: {e}") + return {"error": f"Failed to get agreement metrics: {str(e)}"}, 500 + + def _interpret_alpha(self, alpha: float) -> str: + """Human-readable interpretation of Krippendorff's alpha.""" + if alpha >= 0.8: + return "Good agreement" + elif alpha >= 0.67: + return "Tentative agreement" + elif alpha >= 0.33: + return "Low agreement" + else: + return "Poor agreement" + + def _normalize_annotation_value(self, value: Any) -> Any: + """Normalize annotation value for comparison.""" + if isinstance(value, list): + return tuple(sorted(str(v) for v in value)) + elif isinstance(value, bool): + return str(value).lower() + return str(value) + + def _normalize_categorical_value(self, value: Any) -> Optional[str]: + """Normalize categorical annotation values and label definitions into readable strings.""" + if value is None: + return None + + if isinstance(value, str): + return value + + if isinstance(value, dict): + for key in ("name", "label", "value", "id", "text"): + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + return json.dumps(value, sort_keys=True) + + if isinstance(value, (int, float, bool)): + return str(value) + + return str(value) + + def get_code_cooccurrence_matrix(self, schema_filter: Optional[str] = None, + min_count: int = 1) -> Dict[str, Any]: + """ + Compute pairwise code co-occurrence across instances. + + Two codes co-occur on an instance when at least one annotator applied + each to that instance. Pairs are de-duplicated within an instance + (multiple annotators applying the same pair count once). + + Args: + schema_filter: If set, restrict to codes belonging to this schema. + min_count: Skip pairs with co-occurrence below this threshold. + + Returns: + Dict with `codes` (sorted code list), `pairs` + ({code_a, code_b, count}), and `n_instances` for context. + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + ism = get_item_state_manager() + usm = get_user_state_manager() + users = get_users() + + codes_per_instance: Dict[str, set] = {} + for item in ism.items(): + instance_id = item.get_id() + codes: set = set() + for username in users: + user_state = usm.get_user_state(username) + if not user_state: + continue + all_anns = user_state.get_all_annotations() + if instance_id not in all_anns: + continue + instance_anns = all_anns[instance_id] + labels = instance_anns.get("labels", {}) or {} + for label, value in labels.items(): + schema_name = self._schema_for_label_key(label) + if schema_filter and schema_name != schema_filter: + continue + for code in self._labels_from_value(value): + codes.add(f"{schema_name}::{code}") + spans = instance_anns.get("spans", {}) or {} + for schema_name, span_list in spans.items(): + if schema_filter and schema_name != schema_filter: + continue + for span in span_list or []: + code = span.get("label") or span.get("annotation") + if code: + codes.add(f"{schema_name}::{code}") + if codes: + codes_per_instance[instance_id] = codes + + pair_counts: Dict[Tuple[str, str], int] = {} + for codes in codes_per_instance.values(): + sorted_codes = sorted(codes) + for i in range(len(sorted_codes)): + for j in range(i + 1, len(sorted_codes)): + key = (sorted_codes[i], sorted_codes[j]) + pair_counts[key] = pair_counts.get(key, 0) + 1 + + pairs = [ + {"code_a": a, "code_b": b, "count": c} + for (a, b), c in pair_counts.items() if c >= min_count + ] + pairs.sort(key=lambda x: x["count"], reverse=True) + + all_codes = sorted({ + code for codes in codes_per_instance.values() for code in codes + }) + + return { + "codes": all_codes, + "pairs": pairs, + "n_instances": len(codes_per_instance), + "n_pairs": len(pairs), + "schema_filter": schema_filter, + "min_count": min_count, + } + except Exception as e: + self.logger.error(f"Error computing co-occurrence: {e}") + return {"error": f"Failed to compute co-occurrence: {str(e)}"}, 500 + + def get_code_crosstab(self, attribute_key: str, + schema_filter: Optional[str] = None) -> Dict[str, Any]: + """ + Compute a codes-by-instance-attribute crosstab. + + Each instance contributes one row to the count of (code, attribute_value); + multiple annotators applying the same code count once per instance. + + Args: + attribute_key: Name of the item-metadata field to use as the column axis + (e.g. "site", "condition", "language"). + schema_filter: If set, restrict to codes belonging to this schema. + + Returns: + Dict with `codes` (row labels), `values` (column labels), + `cells` ({code, value, count}), and totals. + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + if not attribute_key: + return {"error": "attribute_key is required"}, 400 + + try: + ism = get_item_state_manager() + usm = get_user_state_manager() + users = get_users() + + cell_counts: Dict[Tuple[str, str], int] = {} + values_seen: set = set() + codes_seen: set = set() + n_instances_with_attr = 0 + + # When the attribute is not on the instance itself, fall + # back to the case-level attribute (cases group instances by + # participant/respondent; the attribute may live on the + # case). No-op for projects that don't use cases. + cb_task_dir = config.get("task_dir", ".") + cb_project = config.get("annotation_task_name") or "default" + + for item in ism.items(): + instance_id = item.get_id() + item_data = self._get_item_data(item) + attr_value = item_data.get(attribute_key) + if attr_value is None or attr_value == "": + try: + from potato.cases import attribute_for_instance + attr_value = attribute_for_instance( + cb_task_dir, cb_project, instance_id, + attribute_key) + except Exception: + attr_value = None + if attr_value is None or attr_value == "": + continue + attr_value = str(attr_value) + values_seen.add(attr_value) + n_instances_with_attr += 1 + + codes: set = set() + for username in users: + user_state = usm.get_user_state(username) + if not user_state: + continue + all_anns = user_state.get_all_annotations() + if instance_id not in all_anns: + continue + instance_anns = all_anns[instance_id] + labels = instance_anns.get("labels", {}) or {} + for label, value in labels.items(): + schema_name = self._schema_for_label_key(label) + if schema_filter and schema_name != schema_filter: + continue + for code in self._labels_from_value(value): + codes.add(f"{schema_name}::{code}") + spans = instance_anns.get("spans", {}) or {} + for schema_name, span_list in spans.items(): + if schema_filter and schema_name != schema_filter: + continue + for span in span_list or []: + code = span.get("label") or span.get("annotation") + if code: + codes.add(f"{schema_name}::{code}") + + for code in codes: + codes_seen.add(code) + key = (code, attr_value) + cell_counts[key] = cell_counts.get(key, 0) + 1 + + cells = [ + {"code": code, "value": value, "count": count} + for (code, value), count in cell_counts.items() + ] + cells.sort(key=lambda x: (x["code"], x["value"])) + + return { + "codes": sorted(codes_seen), + "values": sorted(values_seen), + "cells": cells, + "n_instances": n_instances_with_attr, + "attribute_key": attribute_key, + "schema_filter": schema_filter, + } + except Exception as e: + self.logger.error(f"Error computing crosstab: {e}") + return {"error": f"Failed to compute crosstab: {str(e)}"}, 500 + + @staticmethod + def _schema_for_label_key(label_key) -> str: + """Extract schema name from a label key in user_state annotations.""" + if hasattr(label_key, "schema"): + return label_key.schema + if hasattr(label_key, "get_schema"): + return label_key.get_schema() + return str(label_key) + + @staticmethod + def _labels_from_value(value) -> List[str]: + """Pull individual code names out of an annotation value blob.""" + if value is None or value == "": + return [] + if isinstance(value, dict): + return [k for k, v in value.items() if v] + if isinstance(value, list): + return [str(x) for x in value] + return [str(value)] + + @staticmethod + def _get_item_data(item) -> dict: + """Return the raw data dict for an ItemStateManager item.""" + for attr in ("data", "_data", "item_data"): + data = getattr(item, attr, None) + if isinstance(data, dict): + return data + if hasattr(item, "to_dict"): + try: + return item.to_dict() + except Exception: + pass + return {} + + def get_quality_control_data(self) -> Dict[str, Any]: + """ + Get quality control metrics (attention checks, gold standards, pre-annotation). + + Returns: + Dict containing quality control metrics + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + qc_manager = get_quality_control_manager() + + if not qc_manager: + return { + "enabled": False, + "message": "Quality control not configured" + } + + metrics = qc_manager.get_quality_metrics() + return { + "enabled": True, + **metrics + } + + except Exception as e: + self.logger.error(f"Error getting quality control data: {e}") + return {"error": f"Failed to get quality control data: {str(e)}"}, 500 + + def _behavioral_sequence(self, value: Any) -> list: + """Normalize behavioral list-like values to a safe list.""" + if value is None: + return [] + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + return [value] + + def _behavioral_field(self, payload: Any, field_name: str, default: Any = None) -> Any: + """Read a field from either a dict or an object used in behavioral analytics.""" + if isinstance(payload, dict): + return payload.get(field_name, default) + return getattr(payload, field_name, default) + + def get_behavioral_analytics_data(self) -> Dict[str, Any]: + """ + Get comprehensive behavioral analytics data for all annotators. + + Returns: + Dict containing behavioral analytics metrics including: + - Per-user statistics (time, interactions, AI usage) + - Aggregate statistics + - Quality indicators + - AI assistance analysis + """ + if not self.check_admin_access(): + return {"error": "Admin access required"}, 403 + + try: + usm = get_user_state_manager() + users = get_users() + + user_stats = [] + ai_usage_total = {'requests': 0, 'accepts': 0, 'rejects': 0, 'decision_times': []} + all_times = [] + interaction_counts = Counter() + change_sources = Counter() + total_interactions = 0 + total_changes = 0 + total_ai_requests = 0 + users_with_fast_annotations = 0 + users_with_low_interaction = 0 + users_with_no_changes = 0 + + for user_id in users: + user_state = usm.get_user_state(user_id) + if not user_state: + continue + + behavioral_data = user_state.instance_id_to_behavioral_data + if not behavioral_data: + continue + + user_times = [] + user_interactions = 0 + user_changes = 0 + user_ai_requests = 0 + user_ai_accepts = 0 + user_fast_count = 0 + user_low_interaction_count = 0 + user_no_scroll_count = 0 + user_no_change_count = 0 + + for instance_id, bd in behavioral_data.items(): + time_ms = self._behavioral_field(bd, 'total_time_ms', 0) or 0 + time_sec = time_ms / 1000 + user_times.append(time_sec) + all_times.append(time_sec) + + if time_sec < 5: + user_fast_count += 1 + + interactions = self._behavioral_sequence(self._behavioral_field(bd, 'interactions', [])) + user_interactions += len(interactions) + total_interactions += len(interactions) + if len(interactions) < 3: + user_low_interaction_count += 1 + + for event in interactions: + event_type = self._behavioral_field(event, 'event_type', 'unknown') + interaction_counts[event_type] += 1 + + scroll = self._behavioral_field(bd, 'scroll_depth_max', 0) or 0 + if scroll < 25: + user_no_scroll_count += 1 + + changes = self._behavioral_sequence(self._behavioral_field(bd, 'annotation_changes', [])) + user_changes += len(changes) + total_changes += len(changes) + if len(changes) == 0: + user_no_change_count += 1 + + for change in changes: + source = self._behavioral_field(change, 'source', 'user') + change_sources[source] += 1 + + ai_events = self._behavioral_sequence(self._behavioral_field(bd, 'ai_usage', [])) + for ai in ai_events: + user_ai_requests += 1 + total_ai_requests += 1 + ai_usage_total['requests'] += 1 + + accepted = self._behavioral_field(ai, 'suggestion_accepted', None) + if accepted: + user_ai_accepts += 1 + ai_usage_total['accepts'] += 1 + else: + ai_usage_total['rejects'] += 1 + + decision_time = self._behavioral_field(ai, 'time_to_decision_ms', None) + if isinstance(decision_time, (int, float)): + ai_usage_total['decision_times'].append(decision_time) + + total_instances = len(behavioral_data) + if total_instances > 0: + fast_rate = user_fast_count / total_instances + low_interaction_rate = user_low_interaction_count / total_instances + no_scroll_rate = user_no_scroll_count / total_instances + no_change_rate = user_no_change_count / total_instances + suspicion_score = fast_rate * 0.3 + low_interaction_rate * 0.35 + no_scroll_rate * 0.2 + no_change_rate * 0.15 + + if user_fast_count > 0: + users_with_fast_annotations += 1 + if user_low_interaction_count > 0: + users_with_low_interaction += 1 + if user_no_change_count > 0: + users_with_no_changes += 1 + + user_stats.append({ + 'user_id': user_id, + 'total_instances': total_instances, + 'total_time_sec': sum(user_times), + 'avg_time_sec': sum(user_times) / len(user_times) if user_times else 0, + 'min_time_sec': min(user_times) if user_times else 0, + 'max_time_sec': max(user_times) if user_times else 0, + 'total_interactions': user_interactions, + 'avg_interactions': user_interactions / total_instances, + 'total_changes': user_changes, + 'avg_changes': user_changes / total_instances, + 'ai_requests': user_ai_requests, + 'ai_accepts': user_ai_accepts, + 'ai_accept_rate': (user_ai_accepts / user_ai_requests) if user_ai_requests > 0 else None, + 'fast_annotation_rate': fast_rate, + 'low_interaction_rate': low_interaction_rate, + 'no_scroll_rate': no_scroll_rate, + 'no_change_rate': no_change_rate, + 'suspicion_score': suspicion_score, + 'quality_flag': 'SUSPICIOUS' if suspicion_score > 0.5 else 'WARNING' if suspicion_score > 0.3 else 'OK' + }) + + aggregate = { + 'total_users_with_data': len(user_stats), + 'total_instances': sum(u['total_instances'] for u in user_stats), + 'total_time_minutes': sum(u['total_time_sec'] for u in user_stats) / 60, + 'avg_time_per_instance': sum(all_times) / len(all_times) if all_times else 0, + 'median_time_per_instance': sorted(all_times)[len(all_times)//2] if all_times else 0, + } + aggregate_stats = { + 'total_users': len(user_stats), + 'total_instances': aggregate['total_instances'], + 'avg_time_per_instance_sec': aggregate['avg_time_per_instance'], + 'total_interactions': total_interactions, + 'total_changes': total_changes, + 'total_ai_requests': total_ai_requests, + } + + ai_summary = { + 'total_requests': ai_usage_total['requests'], + 'total_accepts': ai_usage_total['accepts'], + 'total_rejects': ai_usage_total['rejects'], + 'accept_rate': (ai_usage_total['accepts'] / ai_usage_total['requests']) if ai_usage_total['requests'] > 0 else 0, + 'avg_decision_time_ms': sum(ai_usage_total['decision_times']) / len(ai_usage_total['decision_times']) if ai_usage_total['decision_times'] else 0 + } + + flagged_users = [u for u in user_stats if u['quality_flag'] == 'SUSPICIOUS'] + warning_users = [u for u in user_stats if u['quality_flag'] == 'WARNING'] + total_users_with_data = len(user_stats) + quality_summary = { + 'total_flagged': len(flagged_users), + 'total_warnings': len(warning_users), + 'flagged_user_ids': [u['user_id'] for u in flagged_users], + 'warning_user_ids': [u['user_id'] for u in warning_users], + 'high_suspicion_users': len(flagged_users), + 'fast_annotation_rate': (users_with_fast_annotations / total_users_with_data) if total_users_with_data > 0 else 0, + 'low_interaction_rate': (users_with_low_interaction / total_users_with_data) if total_users_with_data > 0 else 0, + 'no_change_rate': (users_with_no_changes / total_users_with_data) if total_users_with_data > 0 else 0, + } + + return { + 'aggregate': aggregate, + 'aggregate_stats': aggregate_stats, + 'ai_usage': ai_summary, + 'quality_summary': quality_summary, + 'interaction_types': dict(interaction_counts.most_common(20)), + 'change_sources': dict(change_sources), + 'users': sorted(user_stats, key=lambda x: -x['suspicion_score']) + } + + except Exception as e: + self.logger.error(f"Error getting behavioral analytics data: {e}") + import traceback + traceback.print_exc() + return {"error": f"Failed to get behavioral analytics data: {str(e)}"}, 500 + + def get_adjudication_overview(self) -> Dict[str, Any]: + """ + Get an overview of adjudication status for the admin dashboard. + + Returns: + Dict with queue stats, adjudicator stats, error taxonomy, + guideline flags, disagreement patterns, and similarity stats. + """ + from potato.adjudication import get_adjudication_manager + + adj_mgr = get_adjudication_manager() + if not adj_mgr or not adj_mgr.adj_config.enabled: + return {"enabled": False, "message": "Adjudication not configured"} + + try: + # Queue stats + queue_stats = adj_mgr.get_stats() + + # Error taxonomy frequency + error_counts = Counter() + guideline_flag_count = 0 + for decision in adj_mgr.decisions.values(): + for tag in decision.error_taxonomy: + error_counts[tag] += 1 + if decision.guideline_update_flag: + guideline_flag_count += 1 + + # Per-adjudicator stats with avg time + adjudicator_details = {} + for adj_id, stats in queue_stats.get("adjudicator_stats", {}).items(): + completed = stats.get("completed", 0) + total_time = stats.get("total_time_ms", 0) + adjudicator_details[adj_id] = { + "completed": completed, + "total_time_ms": total_time, + "avg_time_ms": ( + round(total_time / completed) if completed > 0 else 0 + ), + } + + # Disagreement patterns + disagreement_patterns = self._analyze_disagreement_patterns(adj_mgr) + + # Similarity engine stats + similarity_stats = {} + if adj_mgr.similarity_engine: + similarity_stats = adj_mgr.similarity_engine.get_stats() + + return { + "enabled": True, + "queue_stats": queue_stats, + "adjudicator_details": adjudicator_details, + "error_taxonomy_counts": dict(error_counts.most_common()), + "guideline_flag_count": guideline_flag_count, + "disagreement_patterns": disagreement_patterns, + "similarity_stats": similarity_stats, + } + + except Exception as e: + self.logger.error(f"Error getting adjudication overview: {e}") + return {"enabled": True, "error": str(e)} + + def _analyze_disagreement_patterns(self, adj_mgr) -> List[Dict[str, Any]]: + """ + Analyze per-schema disagreement patterns across the queue. + + Returns: + List of dicts sorted by worst agreement first, with schema name + and average agreement score. + """ + from collections import defaultdict + + schema_scores = defaultdict(list) + + for item in adj_mgr.queue.values(): + for schema_name, score in item.agreement_scores.items(): + schema_scores[schema_name].append(score) + + patterns = [] + for schema_name, scores in schema_scores.items(): + avg = sum(scores) / len(scores) if scores else 1.0 + patterns.append({ + "schema": schema_name, + "avg_agreement": round(avg, 3), + "num_items": len(scores), + }) + + patterns.sort(key=lambda x: x["avg_agreement"]) + return patterns + + # ======================================================================== + # MACE Competence Estimation + # ======================================================================== + + def get_mace_overview(self) -> Dict[str, Any]: + """Get MACE competence estimation overview for admin dashboard. + + Returns: + Dict with competence scores, schema summaries, and config. + """ + from potato.mace_manager import get_mace_manager + + mace_mgr = get_mace_manager() + if not mace_mgr or not mace_mgr.mace_config.enabled: + return {"enabled": False, "message": "MACE not configured"} + + return mace_mgr.get_results_summary() + + def get_mace_predictions( + self, schema: str, instance_id: str = None + ) -> Dict[str, Any]: + """Get MACE predicted labels for a schema, optionally filtered by instance. + + Args: + schema: Schema name to get predictions for. + instance_id: Optional specific instance to filter. + + Returns: + Dict with predictions and entropy data. + """ + from potato.mace_manager import get_mace_manager + + mace_mgr = get_mace_manager() + if not mace_mgr or not mace_mgr.mace_config.enabled: + return {"error": "MACE not configured"} + + return mace_mgr.get_predictions_for_schema(schema, instance_id) + + def _calculate_label_statistics(self, instance_id: str) -> Tuple[Optional[str], float]: + """ + Calculate most frequent label and disagreement for an instance. + + Args: + instance_id: The instance ID to analyze + + Returns: + Tuple of (most_frequent_label, disagreement_score) + """ + try: + usm = get_user_state_manager() + users = get_users() + + all_labels = [] + for username in users: + user_state = usm.get_user_state(username) + if user_state: + annotations = user_state.get_all_annotations() + if instance_id in annotations: + instance_annotations = annotations[instance_id] + if "labels" in instance_annotations: + for label, value in instance_annotations["labels"].items(): + if hasattr(label, 'label_name'): + all_labels.append(label.label_name) + else: + all_labels.append(str(value)) + + if not all_labels: + return None, 0.0 + + label_counts = Counter(all_labels) + most_frequent_label = label_counts.most_common(1)[0][0] + total_annotations = len(all_labels) + most_frequent_count = label_counts[most_frequent_label] + disagreement = 1 - (most_frequent_count / total_annotations) + + return most_frequent_label, disagreement + + except Exception as e: + self.logger.error(f"Error calculating label statistics for instance {instance_id}: {e}") + return None, 0.0 + + +# Global instance +admin_dashboard = AdminDashboard() diff --git a/potato/agent_proxy/__init__.py b/potato/agent_proxy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1cafb201aab49a6197ee08237c2102547ecfaf46 --- /dev/null +++ b/potato/agent_proxy/__init__.py @@ -0,0 +1,44 @@ +""" +Agent Proxy Package + +Provides agent proxy implementations for live agent interaction during annotation. +Proxies communicate with AI agent backends (echo, HTTP, OpenAI) and return +responses to the annotation interface. + +Usage: + from potato.agent_proxy import AgentProxyFactory + + proxy = AgentProxyFactory.create(config) + context = proxy.start_session("Book a flight to Paris") + response = proxy.send_message("Hello", context) +""" + +from .base import AgentMessage, AgentResponse, BaseAgentProxy, AgentProxyFactory +from .session import ( + AgentSession, + AgentSessionManager, + init_agent_session_manager, + get_agent_session_manager, + clear_agent_session_manager, +) +from .sandbox import SafetySandbox, SandboxViolation + +# Import proxy implementations to trigger registration +from . import echo_proxy +from . import http_proxy +from . import openai_proxy +from . import coding_proxy # subprocess_coding + docker_coding + +__all__ = [ + "AgentMessage", + "AgentResponse", + "BaseAgentProxy", + "AgentProxyFactory", + "AgentSession", + "AgentSessionManager", + "init_agent_session_manager", + "get_agent_session_manager", + "clear_agent_session_manager", + "SafetySandbox", + "SandboxViolation", +] diff --git a/potato/agent_proxy/base.py b/potato/agent_proxy/base.py new file mode 100644 index 0000000000000000000000000000000000000000..c03ebda9246d3499b4518681d9e05edb74d46754 --- /dev/null +++ b/potato/agent_proxy/base.py @@ -0,0 +1,138 @@ +""" +Agent Proxy Base Module + +Provides the abstract base class and data structures for agent proxies, +plus a factory registry for creating proxy instances from configuration. + +Agent proxies allow annotators to interact with AI agents live during +annotation tasks. Each proxy type (echo, http, openai) handles +communication with a specific kind of agent backend. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Dict, Any, Optional, List +import logging +import time + +logger = logging.getLogger(__name__) + + +@dataclass +class AgentMessage: + """A single message in an agent conversation.""" + role: str # "user", "agent", "system", "error" + content: str + timestamp: float = field(default_factory=time.time) + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class AgentResponse: + """Response from an agent proxy after sending a message.""" + message: AgentMessage + done: bool = False + error: Optional[str] = None + + +class BaseAgentProxy(ABC): + """ + Abstract base class for agent proxies. + + Subclasses implement communication with specific agent backends + (echo for testing, HTTP for generic REST APIs, OpenAI for chat completions). + """ + + proxy_type: str = "" + + def __init__(self, config: dict): + self.config = config + self._initialize() + + @abstractmethod + def _initialize(self): + """Set up connections, validate config. Called by __init__.""" + pass + + @abstractmethod + def start_session(self, task_description: str) -> dict: + """ + Start a new interaction session. + + Args: + task_description: The task the annotator should accomplish with the agent. + + Returns: + Proxy-specific session context dict (stored in AgentSession.proxy_context). + """ + pass + + @abstractmethod + def send_message(self, message: str, session_context: dict) -> AgentResponse: + """ + Send a message to the agent and get a blocking response. + + Args: + message: The user's message text. + session_context: The proxy-specific context from start_session. + + Returns: + AgentResponse with the agent's reply. + """ + pass + + def end_session(self, session_context: dict): + """ + Clean up session resources. Override if needed. + + Args: + session_context: The proxy-specific context from start_session. + """ + pass + + +class AgentProxyFactory: + """Factory registry for creating agent proxy instances.""" + + _proxies: Dict[str, type] = {} + + @classmethod + def register(cls, proxy_type: str, proxy_class: type): + """Register a proxy type.""" + cls._proxies[proxy_type] = proxy_class + logger.debug(f"Registered agent proxy type: {proxy_type}") + + @classmethod + def create(cls, config: dict) -> BaseAgentProxy: + """ + Create an agent proxy from configuration. + + Args: + config: The full config dict. Reads from config["agent_proxy"]. + + Returns: + Configured BaseAgentProxy instance. + + Raises: + ValueError: If proxy type is unknown or missing. + """ + agent_config = config.get("agent_proxy", {}) + proxy_type = agent_config.get("type") + + if not proxy_type: + raise ValueError("agent_proxy.type is required") + + if proxy_type not in cls._proxies: + supported = ", ".join(sorted(cls._proxies.keys())) + raise ValueError( + f"Unknown agent proxy type: '{proxy_type}'. " + f"Supported types: {supported}" + ) + + proxy_class = cls._proxies[proxy_type] + return proxy_class(agent_config) + + @classmethod + def get_supported_types(cls) -> List[str]: + """Get list of registered proxy type names.""" + return sorted(cls._proxies.keys()) diff --git a/potato/agent_proxy/coding_proxy.py b/potato/agent_proxy/coding_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..883df590eea468aa585a6be0011736ee0115a17c --- /dev/null +++ b/potato/agent_proxy/coding_proxy.py @@ -0,0 +1,466 @@ +""" +Coding-agent proxies โ€” LLM plans + sandboxed code execution. + +Two implementations behind a shared base class: + +- :class:`SubprocessCodingAgentProxy` (default, ``type: subprocess_coding``) + runs each Python or shell action in a per-session temp workspace via + ``subprocess.run`` with a per-step timeout and an output cap. Light + isolation โ€” suitable for trusted-input research workflows. The + workspace is sandboxed (separate cwd) but **not** a security boundary; + malicious code can still touch the host filesystem outside ``cwd``. + +- :class:`DockerCodingAgentProxy` (``type: docker_coding``) runs each + action inside an ephemeral Docker container with ``--network=none``, + ``--memory``, ``--cpus``, ``--read-only`` and a writable workspace + bind-mounted at ``/work``. Real isolation โ€” survives untrusted code. + Requires the ``docker`` Python package and a running Docker daemon. + +Both inherit per-step / per-session / rate-limit enforcement from the +existing :mod:`potato.agent_proxy.sandbox` framework via the standard +``send_message`` flow in ``routes.py:agent_chat_send``. + +Configuration shape (both proxies): + + agent_proxy: + type: subprocess_coding | docker_coding + llm: + endpoint_type: ollama + model: llama3.2:3b + base_url: http://localhost:11434 + temperature: 0.2 + max_tokens: 800 + execution: + per_step_timeout: 8 # seconds + max_output_chars: 4000 + starter_files: {} # {filename: contents} written into workspace + docker: # only for docker_coding + image: python:3.11-slim + memory: 512m + cpus: 1.0 + network: none # "none" or "bridge" + sandbox: { max_steps: 20, ... } # standard agent-proxy sandbox knobs +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import shutil +import subprocess +import tempfile +from abc import abstractmethod +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from .base import AgentMessage, AgentProxyFactory, AgentResponse, BaseAgentProxy + +logger = logging.getLogger(__name__) + + +_PLANNER_SYSTEM_PROMPT = ( + "You are an autonomous coding agent. The user will give you a coding task. " + "Each turn, decide a SINGLE next action and respond with ONLY a JSON object " + "of the form: {\"thought\": str, \"action\": {\"type\": str, \"code\": str}}. " + "The 'type' must be one of:\n" + " - \"python\": run the contents of 'code' as a python script in the workspace\n" + " - \"shell\": run 'code' as a bash command in the workspace\n" + " - \"finish\": stop and return your final answer in 'code' (which is then shown " + "to the user as your conclusion -- no execution happens)\n" + "Keep each action small and focused. Use 'finish' as soon as the task is done." +) + + +@dataclass +class _ExecResult: + stdout: str + stderr: str + exit_code: Optional[int] + timed_out: bool = False + error: Optional[str] = None + + +class CodingAgentProxy(BaseAgentProxy): + """Shared planner/executor scaffold for coding agents. + + Subclasses implement :meth:`_execute` to run an action in their + chosen sandbox. Each call to :meth:`send_message`: + + 1. Appends the user message to the running history. + 2. Asks an LLM (configurable endpoint) for a JSON ``{thought, action}``. + 3. Hands ``action`` to the subclass for execution. + 4. Returns a single reply combining thought + tool output. + """ + + def _initialize(self): + llm_cfg = self.config.get("llm") or {} + self.llm_endpoint_type = llm_cfg.get("endpoint_type", "ollama") + self.llm_model = llm_cfg.get("model") + self.llm_base_url = llm_cfg.get("base_url") + self.llm_temperature = llm_cfg.get("temperature", 0.2) + self.llm_max_tokens = llm_cfg.get("max_tokens", 800) + # OpenAI-compatible servers (vLLM etc.) ignore the key but the SDK + # requires a non-empty string. Ollama needs none. Without forwarding + # this the planner silently failed with "planner_unavailable". + self.llm_api_key = llm_cfg.get("api_key") + # Last endpoint init / call error, surfaced to the user instead of + # an opaque "planner unavailable" message. + self._llm_error: Optional[str] = None + + execution_cfg = self.config.get("execution") or {} + self.per_step_timeout = execution_cfg.get("per_step_timeout", 8) + self.max_output_chars = execution_cfg.get("max_output_chars", 4000) + self.starter_files: Dict[str, str] = execution_cfg.get("starter_files", {}) or {} + + self._llm = None # lazy + + # ------------------------------------------------------------------ + # LLM lazy-init + # ------------------------------------------------------------------ + + def _get_llm(self): + if self._llm is not None: + return self._llm + try: + from potato.ai.ai_endpoint import AIEndpointFactory + + ai_cfg: Dict[str, Any] = { + "model": self.llm_model, + "max_tokens": self.llm_max_tokens, + "temperature": self.llm_temperature, + } + if self.llm_base_url: + ai_cfg["base_url"] = self.llm_base_url + # Forward the key for OpenAI-compatible endpoints; vLLM ignores + # its value but the OpenAI SDK rejects an empty one. Fall back to + # env then a non-empty placeholder so local servers just work. + ai_cfg["api_key"] = ( + self.llm_api_key + or os.environ.get("OPENAI_API_KEY") + or os.environ.get("ANTHROPIC_API_KEY") + or "EMPTY" + ) + + self._llm = AIEndpointFactory.create_endpoint({ + "ai_support": { + "enabled": True, + "endpoint_type": self.llm_endpoint_type, + "ai_config": ai_cfg, + } + }) + self._llm_error = None + except Exception as e: + logger.warning("CodingAgentProxy: planner LLM init failed: %s", e) + self._llm_error = f"{type(e).__name__}: {e}" + self._llm = None + return self._llm + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def start_session(self, task_description: str) -> dict: + workspace = tempfile.mkdtemp(prefix="potato_coding_agent_") + for filename, contents in self.starter_files.items(): + target = os.path.join(workspace, filename) + os.makedirs(os.path.dirname(target) or workspace, exist_ok=True) + with open(target, "w") as f: + f.write(contents) + history = [ + {"role": "system", "content": _PLANNER_SYSTEM_PROMPT}, + { + "role": "system", + "content": f"Workspace: {workspace}\nTask: {task_description}", + }, + ] + return { + "workspace": workspace, + "history": history, + "step": 0, + "finished": False, + } + + def end_session(self, session_context: dict): + workspace = session_context.get("workspace") if session_context else None + if workspace and os.path.isdir(workspace): + shutil.rmtree(workspace, ignore_errors=True) + + # ------------------------------------------------------------------ + # Per-turn flow + # ------------------------------------------------------------------ + + def send_message(self, message: str, session_context: dict) -> AgentResponse: + if session_context.get("finished"): + return AgentResponse( + message=AgentMessage(role="agent", content="(session already finished)"), + done=True, + ) + + history: List[Dict[str, str]] = session_context.setdefault("history", []) + history.append({"role": "user", "content": message}) + session_context["step"] = session_context.get("step", 0) + 1 + + plan = self._plan_next_action(history) + if plan is None: + detail = self._llm_error or "no response from planner LLM" + reply = ( + f"Planner LLM unavailable ({self.llm_endpoint_type}): " + f"{detail}" + ) + history.append({"role": "assistant", "content": reply}) + session_context["finished"] = True + return AgentResponse( + message=AgentMessage(role="error", content=reply), error="planner_unavailable", + ) + + thought = plan.get("thought", "") + action = plan.get("action") or {} + atype = (action.get("type") or "").strip().lower() + code = action.get("code") or "" + + if atype == "finish": + reply = self._format_finish_reply(thought, code) + history.append({"role": "assistant", "content": reply}) + session_context["finished"] = True + return AgentResponse( + message=AgentMessage(role="agent", content=reply), done=True, + ) + + if atype not in ("python", "shell"): + reply = ( + f"{thought}\n\n[invalid action type {atype!r}; try 'python', " + "'shell', or 'finish']" + ) + history.append({"role": "assistant", "content": reply}) + return AgentResponse(message=AgentMessage(role="agent", content=reply)) + + result = self._execute(atype, code, session_context) + reply = self._format_exec_reply(thought, atype, code, result) + history.append({"role": "assistant", "content": reply}) + return AgentResponse(message=AgentMessage(role="agent", content=reply)) + + # ------------------------------------------------------------------ + # Planning + # ------------------------------------------------------------------ + + def _plan_next_action(self, history: List[Dict[str, str]]) -> Optional[Dict[str, Any]]: + endpoint = self._get_llm() + if endpoint is None: + return None + try: + if hasattr(endpoint, "chat_query"): + raw = endpoint.chat_query(history) + else: + flat = "\n".join(f'{m["role"]}: {m["content"]}' for m in history) + raw = endpoint.query(flat + "\nassistant:", None) + except Exception as e: + logger.warning("Planner LLM call failed: %s", e) + self._llm_error = f"{type(e).__name__}: {e}" + return None + + if isinstance(raw, dict): + return raw # already parsed JSON + text = str(raw or "").strip() + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if match: + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + pass + # Last-ditch: treat the whole text as a finish reply. + return {"thought": "", "action": {"type": "finish", "code": text[:500]}} + + # ------------------------------------------------------------------ + # Reply formatting + # ------------------------------------------------------------------ + + def _format_exec_reply( + self, thought: str, atype: str, code: str, result: _ExecResult + ) -> str: + parts: List[str] = [] + if thought: + parts.append(thought.strip()) + parts.append(f"```{atype}\n{code.strip()}\n```") + out_block: List[str] = [] + if result.timed_out: + out_block.append(f"[timeout after {self.per_step_timeout}s]") + if result.error: + out_block.append(f"[error: {result.error}]") + if result.exit_code is not None: + out_block.append(f"[exit={result.exit_code}]") + if result.stdout: + out_block.append("stdout:\n" + self._truncate(result.stdout)) + if result.stderr: + out_block.append("stderr:\n" + self._truncate(result.stderr)) + if not out_block: + out_block.append("(no output)") + parts.append("\n".join(out_block)) + return "\n\n".join(parts) + + def _format_finish_reply(self, thought: str, final_text: str) -> str: + parts = [] + if thought: + parts.append(thought.strip()) + if final_text: + parts.append(final_text.strip()) + if not parts: + parts.append("(done)") + return "\n\n".join(parts) + + def _truncate(self, text: str) -> str: + if len(text) <= self.max_output_chars: + return text + cut = self.max_output_chars + return text[:cut] + f"\n[...truncated {len(text) - cut} chars]" + + # ------------------------------------------------------------------ + # Sandbox-specific execution + # ------------------------------------------------------------------ + + @abstractmethod + def _execute( + self, action_type: str, code: str, session_context: dict + ) -> _ExecResult: + """Execute ``code`` (one of 'python' / 'shell') in the sandbox.""" + + +class SubprocessCodingAgentProxy(CodingAgentProxy): + """Local subprocess-based execution. + + NOT a security boundary -- the per-step timeout + tempdir cwd is the + only protection. Use ``DockerCodingAgentProxy`` for untrusted input. + """ + + proxy_type = "subprocess_coding" + + def _execute( + self, action_type: str, code: str, session_context: dict + ) -> _ExecResult: + workspace = session_context["workspace"] + env = self._build_env() + try: + if action_type == "python": + script_path = os.path.join(workspace, "_action.py") + with open(script_path, "w") as f: + f.write(code) + proc = subprocess.run( + ["python", script_path], + cwd=workspace, + env=env, + capture_output=True, + text=True, + timeout=self.per_step_timeout, + ) + else: # shell + proc = subprocess.run( + ["bash", "-c", code], + cwd=workspace, + env=env, + capture_output=True, + text=True, + timeout=self.per_step_timeout, + ) + except subprocess.TimeoutExpired as e: + return _ExecResult( + stdout=(e.stdout or "") if isinstance(e.stdout, str) else "", + stderr=(e.stderr or "") if isinstance(e.stderr, str) else "", + exit_code=None, + timed_out=True, + ) + except FileNotFoundError as e: + return _ExecResult(stdout="", stderr="", exit_code=None, error=str(e)) + return _ExecResult( + stdout=proc.stdout or "", + stderr=proc.stderr or "", + exit_code=proc.returncode, + ) + + def _build_env(self) -> Dict[str, str]: + # Strip env down to a minimal set so subprocess code can't + # accidentally exfiltrate the host's secrets via env vars. + keep = {"PATH", "HOME", "LANG", "LC_ALL"} + return {k: v for k, v in os.environ.items() if k in keep} + + +class DockerCodingAgentProxy(CodingAgentProxy): + """Ephemeral-container execution. Real isolation; requires Docker.""" + + proxy_type = "docker_coding" + + def _initialize(self): + super()._initialize() + docker_cfg = self.config.get("docker") or {} + self.docker_image = docker_cfg.get("image", "python:3.11-slim") + self.docker_memory = docker_cfg.get("memory", "512m") + self.docker_cpus = str(docker_cfg.get("cpus", 1.0)) + self.docker_network = docker_cfg.get("network", "none") + self._docker = None # lazy + # Sanity check: warn if docker CLI isn't on PATH + if shutil.which("docker") is None: + logger.warning( + "DockerCodingAgentProxy: 'docker' CLI not found on PATH. " + "Container execution will fail at runtime." + ) + + def _execute( + self, action_type: str, code: str, session_context: dict + ) -> _ExecResult: + workspace = session_context["workspace"] + # Materialise the code as a file in the workspace so the container + # can run it without inline injection through `-c`. + if action_type == "python": + target = os.path.join(workspace, "_action.py") + with open(target, "w") as f: + f.write(code) + container_cmd = ["python", "/work/_action.py"] + else: # shell + target = os.path.join(workspace, "_action.sh") + with open(target, "w") as f: + f.write(code) + os.chmod(target, 0o755) + container_cmd = ["bash", "/work/_action.sh"] + + cmd = [ + "docker", "run", "--rm", + f"--network={self.docker_network}", + f"--memory={self.docker_memory}", + f"--cpus={self.docker_cpus}", + "--read-only", + "--tmpfs", "/tmp:exec,size=64m", + "-v", f"{workspace}:/work", + "-w", "/work", + self.docker_image, + ] + container_cmd + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=self.per_step_timeout + 5, # docker pull/start overhead + ) + except subprocess.TimeoutExpired as e: + return _ExecResult( + stdout=(e.stdout or "") if isinstance(e.stdout, str) else "", + stderr=(e.stderr or "") if isinstance(e.stderr, str) else "", + exit_code=None, + timed_out=True, + ) + except FileNotFoundError as e: + return _ExecResult(stdout="", stderr="", exit_code=None, error=str(e)) + return _ExecResult( + stdout=proc.stdout or "", + stderr=proc.stderr or "", + exit_code=proc.returncode, + ) + + +# Register both with the factory so configs can refer to them by name. +AgentProxyFactory.register("subprocess_coding", SubprocessCodingAgentProxy) +AgentProxyFactory.register("docker_coding", DockerCodingAgentProxy) diff --git a/potato/agent_proxy/echo_proxy.py b/potato/agent_proxy/echo_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..c6d44e4f3e229536090db500502f8fcee095f74f --- /dev/null +++ b/potato/agent_proxy/echo_proxy.py @@ -0,0 +1,55 @@ +""" +Echo Agent Proxy + +A testing/demo proxy that returns responses from a configurable list. +Cycles through responses in order, wrapping around when exhausted. + +Configuration: + agent_proxy: + type: echo + responses: + - "I understand your request." + - "Working on it now." + - "Here's what I found." +""" + +import logging + +from .base import BaseAgentProxy, AgentMessage, AgentResponse, AgentProxyFactory + +logger = logging.getLogger(__name__) + + +class EchoProxy(BaseAgentProxy): + """Test proxy that returns canned responses in order.""" + + proxy_type = "echo" + + def _initialize(self): + self.responses = self.config.get("responses", [ + "I understand.", + "Working on it.", + "Done!", + ]) + + def start_session(self, task_description: str) -> dict: + return {"response_index": 0, "task_description": task_description} + + def send_message(self, message: str, session_context: dict) -> AgentResponse: + idx = session_context.get("response_index", 0) + response_text = self.responses[idx % len(self.responses)] + session_context["response_index"] = idx + 1 + + return AgentResponse( + message=AgentMessage( + role="agent", + content=response_text, + ) + ) + + def end_session(self, session_context: dict): + pass + + +# Register with factory +AgentProxyFactory.register("echo", EchoProxy) diff --git a/potato/agent_proxy/http_proxy.py b/potato/agent_proxy/http_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..59315a0e5b46b1ed438adc567365d44d75486441 --- /dev/null +++ b/potato/agent_proxy/http_proxy.py @@ -0,0 +1,108 @@ +""" +Generic HTTP Agent Proxy + +POSTs to any REST endpoint with configurable field mapping. +Supports sending full conversation history and custom headers. + +Configuration: + agent_proxy: + type: http + url: "http://localhost:8080/chat" + headers: + Authorization: "Bearer YOUR_KEY" + message_key: "message" # key in request body for user message + response_key: "response" # key in response JSON for agent reply + session_id_key: "session_id" # key in request/response for session tracking + send_history: false # whether to send full conversation history + history_key: "messages" # key for history array in request body +""" + +import logging +import uuid + +import requests + +from .base import BaseAgentProxy, AgentMessage, AgentResponse, AgentProxyFactory + +logger = logging.getLogger(__name__) + + +class GenericHTTPProxy(BaseAgentProxy): + """Generic REST API proxy with configurable field mapping.""" + + proxy_type = "http" + + def _initialize(self): + self.url = self.config.get("url") + if not self.url: + raise ValueError("http proxy requires 'url' in agent_proxy config") + + self.headers = self.config.get("headers", {}) + self.message_key = self.config.get("message_key", "message") + self.response_key = self.config.get("response_key", "response") + self.session_id_key = self.config.get("session_id_key", "session_id") + self.send_history = self.config.get("send_history", False) + self.history_key = self.config.get("history_key", "messages") + self.timeout = self.config.get("sandbox", {}).get( + "request_timeout_seconds", 60 + ) + + def start_session(self, task_description: str) -> dict: + return { + "session_id": str(uuid.uuid4()), + "task_description": task_description, + "history": [], + } + + def send_message(self, message: str, session_context: dict) -> AgentResponse: + payload = { + self.message_key: message, + self.session_id_key: session_context["session_id"], + } + + if self.send_history: + payload[self.history_key] = session_context.get("history", []) + + try: + resp = requests.post( + self.url, + json=payload, + headers=self.headers, + timeout=self.timeout, + ) + resp.raise_for_status() + data = resp.json() + + response_text = data.get(self.response_key, "") + if not response_text and isinstance(data, str): + response_text = data + + # Update history + session_context.setdefault("history", []).append( + {"role": "user", "content": message} + ) + session_context["history"].append( + {"role": "agent", "content": response_text} + ) + + return AgentResponse( + message=AgentMessage(role="agent", content=str(response_text)) + ) + + except requests.Timeout: + return AgentResponse( + message=AgentMessage(role="error", content="Agent request timed out."), + error="timeout", + ) + except requests.RequestException as e: + logger.error(f"HTTP proxy request failed: {e}") + return AgentResponse( + message=AgentMessage( + role="error", content=f"Agent communication error: {e}" + ), + error=str(e), + ) + + +# Register with factory +AgentProxyFactory.register("http", GenericHTTPProxy) diff --git a/potato/agent_proxy/openai_proxy.py b/potato/agent_proxy/openai_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..ae5a42136b79a7e01f26e7f6c009634e8d205261 --- /dev/null +++ b/potato/agent_proxy/openai_proxy.py @@ -0,0 +1,105 @@ +""" +OpenAI Chat Completions Agent Proxy + +Uses the OpenAI SDK to communicate with chat completion models. +Maintains conversation history in session context for multi-turn dialogue. + +Configuration: + agent_proxy: + type: openai + api_key: "${OPENAI_API_KEY}" # or set OPENAI_API_KEY env var + model: "gpt-4o" + system_prompt: "You are a helpful travel agent." + temperature: 0.7 + max_tokens: 1024 +""" + +import logging +import os + +from .base import BaseAgentProxy, AgentMessage, AgentResponse, AgentProxyFactory + +logger = logging.getLogger(__name__) + + +class OpenAIChatProxy(BaseAgentProxy): + """OpenAI Chat Completions proxy.""" + + proxy_type = "openai" + + def _initialize(self): + api_key = self.config.get("api_key", "") + # Support environment variable references like ${OPENAI_API_KEY} + if api_key.startswith("${") and api_key.endswith("}"): + env_var = api_key[2:-1] + api_key = os.environ.get(env_var, "") + + if not api_key: + api_key = os.environ.get("OPENAI_API_KEY", "") + + if not api_key: + raise ValueError( + "OpenAI proxy requires api_key in config or OPENAI_API_KEY env var" + ) + + try: + import openai + self.client = openai.OpenAI(api_key=api_key) + except ImportError: + raise ImportError( + "openai package is required for the OpenAI proxy. " + "Install with: pip install openai" + ) + + self.model = self.config.get("model", "gpt-4o") + self.system_prompt = self.config.get("system_prompt", "") + self.temperature = self.config.get("temperature", 0.7) + self.max_tokens = self.config.get("max_tokens", 1024) + self.timeout = self.config.get("sandbox", {}).get( + "request_timeout_seconds", 60 + ) + + def start_session(self, task_description: str) -> dict: + messages = [] + if self.system_prompt: + messages.append({"role": "system", "content": self.system_prompt}) + # Include task description as system context + messages.append({ + "role": "system", + "content": f"The user's task: {task_description}", + }) + return {"messages": messages} + + def send_message(self, message: str, session_context: dict) -> AgentResponse: + messages = session_context.get("messages", []) + messages.append({"role": "user", "content": message}) + + try: + response = self.client.chat.completions.create( + model=self.model, + messages=messages, + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + + content = response.choices[0].message.content or "" + messages.append({"role": "assistant", "content": content}) + session_context["messages"] = messages + + return AgentResponse( + message=AgentMessage(role="agent", content=content) + ) + + except Exception as e: + logger.error(f"OpenAI proxy error: {e}") + return AgentResponse( + message=AgentMessage( + role="error", content=f"Agent error: {e}" + ), + error=str(e), + ) + + +# Register with factory +AgentProxyFactory.register("openai", OpenAIChatProxy) diff --git a/potato/agent_proxy/sandbox.py b/potato/agent_proxy/sandbox.py new file mode 100644 index 0000000000000000000000000000000000000000..ae7b0f0763fe56d8044feb77020f1d49a8a4300d --- /dev/null +++ b/potato/agent_proxy/sandbox.py @@ -0,0 +1,76 @@ +""" +Agent Proxy Safety Sandbox + +Enforces limits on agent interactions: step counts, session timeouts, +rate limits, and request timeouts. Prevents runaway or abusive sessions. +""" + +import time +import threading +import logging +from collections import defaultdict +from typing import Dict, List + +logger = logging.getLogger(__name__) + + +class SandboxViolation(Exception): + """Raised when a safety limit is exceeded.""" + pass + + +class SafetySandbox: + """Enforces safety limits on agent interactions.""" + + def __init__(self, config: dict): + sandbox_config = config.get("sandbox", {}) + self.max_steps = sandbox_config.get("max_steps", 20) + self.max_session_seconds = sandbox_config.get("max_session_seconds", 600) + self.rate_limit_per_minute = sandbox_config.get("rate_limit_per_minute", 10) + self.request_timeout = sandbox_config.get("request_timeout_seconds", 60) + + # Sliding window rate limit tracking: user_id -> list of timestamps + self._rate_windows: Dict[str, List[float]] = defaultdict(list) + self._lock = threading.Lock() + + def check_step_limit(self, current_steps: int): + """Raise SandboxViolation if step limit reached.""" + if current_steps >= self.max_steps: + raise SandboxViolation( + f"Step limit reached ({self.max_steps}). " + f"Please finish the conversation." + ) + + def check_session_timeout(self, session_start: float): + """Raise SandboxViolation if session has timed out.""" + elapsed = time.time() - session_start + if elapsed > self.max_session_seconds: + raise SandboxViolation( + f"Session timeout ({self.max_session_seconds}s). " + f"Please finish the conversation." + ) + + def check_rate_limit(self, user_id: str): + """Raise SandboxViolation if user is sending too fast.""" + now = time.time() + window_start = now - 60.0 + + with self._lock: + # Remove old entries outside the 1-minute window + timestamps = self._rate_windows[user_id] + self._rate_windows[user_id] = [ + t for t in timestamps if t > window_start + ] + + if len(self._rate_windows[user_id]) >= self.rate_limit_per_minute: + raise SandboxViolation( + f"Rate limit exceeded ({self.rate_limit_per_minute}/min). " + f"Please wait before sending another message." + ) + + # Record this request + self._rate_windows[user_id].append(now) + + def get_request_timeout(self) -> float: + """Get the timeout in seconds for proxy HTTP requests.""" + return self.request_timeout diff --git a/potato/agent_proxy/session.py b/potato/agent_proxy/session.py new file mode 100644 index 0000000000000000000000000000000000000000..9d5e83a7d438ca3b29cb371ff1b49d6c40f1da2d --- /dev/null +++ b/potato/agent_proxy/session.py @@ -0,0 +1,119 @@ +""" +Agent Session Manager + +Thread-safe singleton that tracks active agent interaction sessions. +Each session maps a (user_id, instance_id) pair to an AgentSession +containing the proxy, conversation history, and step count. + +Follows the same singleton pattern as ItemStateManager and UserStateManager. +""" + +import threading +import time +import logging +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + +from .base import AgentMessage, BaseAgentProxy + +logger = logging.getLogger(__name__) + + +@dataclass +class AgentSession: + """An active agent interaction session.""" + user_id: str + instance_id: str + proxy: BaseAgentProxy + task_description: str + proxy_context: dict = field(default_factory=dict) + messages: List[AgentMessage] = field(default_factory=list) + step_count: int = 0 + started_at: float = field(default_factory=time.time) + finished: bool = False + + +class AgentSessionManager: + """Thread-safe manager for active agent sessions.""" + + def __init__(self, config: dict): + self.config = config + self._sessions: Dict[Tuple[str, str], AgentSession] = {} + self._lock = threading.RLock() + + def create_session( + self, + user_id: str, + instance_id: str, + proxy: BaseAgentProxy, + task_description: str, + ) -> AgentSession: + """Create a new session for a user/instance pair.""" + with self._lock: + key = (user_id, instance_id) + if key in self._sessions and not self._sessions[key].finished: + logger.warning( + f"Session already exists for {key}, returning existing" + ) + return self._sessions[key] + + proxy_context = proxy.start_session(task_description) + session = AgentSession( + user_id=user_id, + instance_id=instance_id, + proxy=proxy, + task_description=task_description, + proxy_context=proxy_context, + ) + self._sessions[key] = session + logger.debug(f"Created agent session for {key}") + return session + + def get_session( + self, user_id: str, instance_id: str + ) -> Optional[AgentSession]: + """Get an active session, or None if not found.""" + with self._lock: + return self._sessions.get((user_id, instance_id)) + + def remove_session(self, user_id: str, instance_id: str): + """Remove a session and clean up proxy resources.""" + with self._lock: + key = (user_id, instance_id) + session = self._sessions.pop(key, None) + if session: + try: + session.proxy.end_session(session.proxy_context) + except Exception as e: + logger.warning(f"Error ending proxy session for {key}: {e}") + logger.debug(f"Removed agent session for {key}") + + +# Singleton management +_AGENT_SESSION_MANAGER: Optional[AgentSessionManager] = None +_AGENT_SESSION_MANAGER_LOCK = threading.Lock() + + +def init_agent_session_manager(config: dict) -> AgentSessionManager: + """Initialize the singleton AgentSessionManager.""" + global _AGENT_SESSION_MANAGER + if _AGENT_SESSION_MANAGER is None: + with _AGENT_SESSION_MANAGER_LOCK: + if _AGENT_SESSION_MANAGER is None: + _AGENT_SESSION_MANAGER = AgentSessionManager(config) + return _AGENT_SESSION_MANAGER + + +def get_agent_session_manager() -> AgentSessionManager: + """Get the singleton AgentSessionManager.""" + global _AGENT_SESSION_MANAGER + if _AGENT_SESSION_MANAGER is None: + raise ValueError("AgentSessionManager has not been initialized yet!") + return _AGENT_SESSION_MANAGER + + +def clear_agent_session_manager(): + """Clear the singleton instance (for testing).""" + global _AGENT_SESSION_MANAGER + with _AGENT_SESSION_MANAGER_LOCK: + _AGENT_SESSION_MANAGER = None diff --git a/potato/agent_runner.py b/potato/agent_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..760a6e85fbf8c878b85feb0051c5846101bb13f6 --- /dev/null +++ b/potato/agent_runner.py @@ -0,0 +1,1008 @@ +""" +Live Agent Runner + +Manages an AI agent that browses the web via Playwright, controlled by an LLM. +Annotators can observe, pause, instruct, or take over the agent in real time. + +The agent loop runs in a background thread with its own asyncio event loop. +Communication with Flask routes happens through thread-safe state and queues. +""" + +import asyncio +import base64 +import json +import logging +import os +import threading +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from queue import Queue, Empty +from typing import Any, Callable, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +class AgentState(Enum): + """States of the agent lifecycle.""" + IDLE = "idle" + RUNNING = "running" + PAUSED = "paused" + TAKEOVER = "takeover" + COMPLETED = "completed" + ERROR = "error" + + +@dataclass +class AgentStep: + """A single step in the agent's execution.""" + step_index: int + screenshot_path: str + action: Dict[str, Any] + thought: str + observation: str + timestamp: float + url: str = "" + viewport: Optional[Dict[str, int]] = None + coordinates: Optional[Dict[str, int]] = None + element: Optional[Dict[str, Any]] = None + annotator_instruction: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + d = { + "step_index": self.step_index, + "screenshot_url": self.screenshot_path, + "action_type": self.action.get("type", "unknown"), + "action": self.action, + "thought": self.thought, + "observation": self.observation, + "timestamp": self.timestamp, + "url": self.url, + } + if self.viewport: + d["viewport"] = self.viewport + if self.coordinates: + d["coordinates"] = self.coordinates + if self.element: + d["element"] = self.element + if self.annotator_instruction: + d["annotator_instruction"] = self.annotator_instruction + return d + + +@dataclass +class AgentConfig: + """Configuration for the agent runner.""" + max_steps: int = 30 + step_delay: float = 1.0 + viewport_width: int = 1280 + viewport_height: int = 720 + system_prompt: str = "" + model: str = "claude-sonnet-4-20250514" + api_key: str = "" + max_tokens: int = 4096 + temperature: float = 0.3 + endpoint_type: str = "anthropic_vision" + history_window: int = 5 # Number of recent steps to include in LLM context + timeout: int = 60 # Per-request timeout in seconds + + base_url: str = "" # For Ollama: server URL + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "AgentConfig": + """Create AgentConfig from a live_agent YAML config dict.""" + ai_config = config.get("ai_config", {}) + viewport = config.get("viewport", {}) + endpoint_type = config.get("endpoint_type", "anthropic_vision") + + # API key: Ollama doesn't need one; OpenAI-compatible servers + # (e.g. vLLM) ignore it but the SDK requires a non-empty string. + if endpoint_type == "ollama_vision": + api_key = ai_config.get("api_key", "") + default_model = "gemma3:4b" + elif endpoint_type == "openai_vision": + api_key = ai_config.get("api_key", os.environ.get("OPENAI_API_KEY", "EMPTY")) + default_model = "" # must be set explicitly (e.g. served model id) + else: + api_key = ai_config.get("api_key", os.environ.get("ANTHROPIC_API_KEY", "")) + default_model = "claude-sonnet-4-20250514" + + return cls( + max_steps=config.get("max_steps", 30), + step_delay=config.get("step_delay", 1.0), + viewport_width=viewport.get("width", 1280), + viewport_height=viewport.get("height", 720), + system_prompt=config.get("system_prompt", DEFAULT_SYSTEM_PROMPT), + model=ai_config.get("model", default_model), + api_key=api_key, + max_tokens=ai_config.get("max_tokens", 4096), + temperature=ai_config.get("temperature", 0.3), + endpoint_type=endpoint_type, + history_window=config.get("history_window", 5), + timeout=ai_config.get("timeout", 60), + base_url=ai_config.get("base_url", "http://localhost:11434"), + ) + + +DEFAULT_SYSTEM_PROMPT = """You are a web browsing agent. You can see screenshots of web pages and take actions to complete tasks. + +For each step, analyze the current screenshot and respond with a JSON object: +{ + "thought": "Your reasoning about what you see and what to do next", + "action": { + "type": "click|type|scroll|navigate|wait|done", + // For click: "x": 100, "y": 200 + // For type: "text": "hello world" + // For scroll: "direction": "up|down", "amount": 300 + // For navigate: "url": "https://..." + // For wait: (no extra fields) + // For done: "summary": "Task completed because..." + } +} + +Always respond with valid JSON only. No markdown, no extra text.""" + + +class AgentRunner: + """ + Runs an AI agent that browses the web via Playwright. + + The agent loop: + 1. Takes a screenshot + 2. Sends it to the LLM with context/history + 3. Parses the LLM response for an action + 4. Executes the action via Playwright + 5. Emits events to all listeners (for SSE) + 6. Repeats until done, error, or max_steps + + Thread-safe control methods allow pause/resume/instruct/takeover. + """ + + def __init__(self, session_id: str, config: AgentConfig, screenshot_dir: str): + self.session_id = session_id + self.config = config + self.screenshot_dir = screenshot_dir + + # State + self._state = AgentState.IDLE + self._state_lock = threading.Lock() + self._steps: List[AgentStep] = [] + self._error: Optional[str] = None + + # Control + self._pause_event = threading.Event() + self._pause_event.set() # Not paused initially + self._stop_flag = threading.Event() + self._instruction_queue: Queue = Queue() + self._takeover_actions: Queue = Queue() + + # Listeners for SSE + self._listeners: List[Callable] = [] + self._listeners_lock = threading.Lock() + + # Annotator interactions log + self._interactions: List[Dict[str, Any]] = [] + + # Playwright session (set during run) + self._playwright_session = None + self._llm_client = None + + # Background thread + self._thread: Optional[threading.Thread] = None + + @property + def state(self) -> AgentState: + with self._state_lock: + return self._state + + @state.setter + def state(self, new_state: AgentState): + with self._state_lock: + old_state = self._state + self._state = new_state + self._emit_event("state_change", { + "old_state": old_state.value, + "new_state": new_state.value, + "timestamp": time.time(), + }) + + @property + def steps(self) -> List[AgentStep]: + return list(self._steps) + + @property + def step_count(self) -> int: + return len(self._steps) + + @property + def error(self) -> Optional[str]: + return self._error + + # --- Control methods (thread-safe) --- + + def pause(self): + """Pause the agent loop after the current step completes.""" + if self.state == AgentState.RUNNING: + self._pause_event.clear() + self.state = AgentState.PAUSED + logger.info(f"[{self.session_id}] Agent paused") + + def resume(self): + """Resume a paused agent.""" + if self.state == AgentState.PAUSED: + self.state = AgentState.RUNNING + self._pause_event.set() + logger.info(f"[{self.session_id}] Agent resumed") + + def inject_instruction(self, instruction: str): + """Send an instruction to the agent (processed at next step).""" + self._instruction_queue.put(instruction) + self._interactions.append({ + "type": "instruction", + "text": instruction, + "timestamp": time.time(), + "step_index": self.step_count, + }) + self._emit_event("instruction_received", {"instruction": instruction}) + logger.info(f"[{self.session_id}] Instruction injected: {instruction[:100]}") + + def enter_takeover(self): + """Switch to manual takeover mode.""" + if self.state in (AgentState.RUNNING, AgentState.PAUSED): + self._pause_event.clear() # Pause the agent loop + self.state = AgentState.TAKEOVER + self._interactions.append({ + "type": "takeover_start", + "timestamp": time.time(), + "step_index": self.step_count, + }) + logger.info(f"[{self.session_id}] Takeover mode entered") + + def exit_takeover(self): + """Exit manual takeover and resume the agent.""" + if self.state == AgentState.TAKEOVER: + self._interactions.append({ + "type": "takeover_end", + "timestamp": time.time(), + "step_index": self.step_count, + }) + self.state = AgentState.RUNNING + self._pause_event.set() + logger.info(f"[{self.session_id}] Takeover mode exited") + + def submit_manual_action(self, action: Dict[str, Any]): + """Submit a manual action during takeover mode.""" + if self.state == AgentState.TAKEOVER: + self._takeover_actions.put(action) + + def stop(self): + """Stop the agent loop.""" + self._stop_flag.set() + self._pause_event.set() # Unblock if paused + logger.info(f"[{self.session_id}] Stop requested") + + # --- Listener management --- + + def add_listener(self, callback: Callable): + """Add an SSE listener callback.""" + with self._listeners_lock: + self._listeners.append(callback) + + def remove_listener(self, callback: Callable): + """Remove an SSE listener callback.""" + with self._listeners_lock: + self._listeners = [l for l in self._listeners if l is not callback] + + def _emit_event(self, event_type: str, data: Dict[str, Any]): + """Emit an event to all listeners.""" + event = {"type": event_type, "data": data, "session_id": self.session_id} + with self._listeners_lock: + for listener in self._listeners: + try: + listener(event) + except Exception as e: + logger.warning(f"Listener error: {e}") + + # --- Main agent loop --- + + def start(self, task_description: str, start_url: str): + """Start the agent in a background thread.""" + if self.state != AgentState.IDLE: + raise RuntimeError(f"Cannot start agent in state {self.state}") + + self._thread = threading.Thread( + target=self._run_thread, + args=(task_description, start_url), + daemon=True, + name=f"agent-{self.session_id}", + ) + self._thread.start() + + def _run_thread(self, task_description: str, start_url: str): + """Thread target: runs the async agent loop.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete(self._run_async(task_description, start_url)) + except Exception as e: + logger.error(f"[{self.session_id}] Agent thread error: {e}") + self._error = str(e) + self.state = AgentState.ERROR + self._emit_event("error", {"message": str(e)}) + finally: + loop.close() + + async def _run_async(self, task_description: str, start_url: str): + """Async agent loop.""" + from potato.web_playwright import PlaywrightSession + + self.state = AgentState.RUNNING + + # Initialize Playwright + self._playwright_session = PlaywrightSession( + width=self.config.viewport_width, + height=self.config.viewport_height, + ) + started = await self._playwright_session.start(start_url) + if not started: + raise RuntimeError("Failed to start Playwright browser session") + + # Initialize LLM client + self._init_llm_client() + + self._emit_event("started", { + "task": task_description, + "start_url": start_url, + "max_steps": self.config.max_steps, + }) + + try: + for step_index in range(self.config.max_steps): + # Check stop flag + if self._stop_flag.is_set(): + logger.info(f"[{self.session_id}] Stopped by user") + break + + # Wait if paused (blocks until resume/stop) + while not self._pause_event.is_set(): + if self._stop_flag.is_set(): + break + # Handle takeover actions while paused in takeover mode + if self.state == AgentState.TAKEOVER: + await self._process_takeover_actions() + await asyncio.sleep(0.1) + + if self._stop_flag.is_set(): + break + + # Check for injected instructions + instruction = None + try: + instruction = self._instruction_queue.get_nowait() + except Empty: + pass + + # Execute one agent step + step = await self._agent_step( + step_index, task_description, instruction + ) + self._steps.append(step) + + # Check if agent decided it's done + if step.action.get("type") == "done": + logger.info(f"[{self.session_id}] Agent completed task") + break + + # Step delay + if self.config.step_delay > 0: + await asyncio.sleep(self.config.step_delay) + + self.state = AgentState.COMPLETED + self._emit_event("complete", { + "total_steps": len(self._steps), + "final_url": (await self._playwright_session.get_state()).get("url", ""), + }) + + finally: + await self._playwright_session.stop() + self._playwright_session = None + + async def _agent_step( + self, + step_index: int, + task_description: str, + instruction: Optional[str] = None, + ) -> AgentStep: + """Execute a single agent step: screenshot โ†’ LLM โ†’ action โ†’ emit.""" + + # 1. Take screenshot + screenshot_bytes = await self._playwright_session.screenshot() + if not screenshot_bytes: + raise RuntimeError("Failed to capture screenshot") + + screenshot_path = os.path.join( + self.screenshot_dir, f"step_{step_index:03d}.png" + ) + os.makedirs(os.path.dirname(screenshot_path), exist_ok=True) + with open(screenshot_path, "wb") as f: + f.write(screenshot_bytes) + + # 2. Get page state + page_state = await self._playwright_session.get_state() + + # 3. Emit thinking event + self._emit_event("thinking", { + "step_index": step_index, + "screenshot_url": screenshot_path, + "url": page_state.get("url", ""), + }) + + # 4. Build messages and query LLM + screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8") + messages = self._build_llm_messages( + screenshot_b64, task_description, instruction + ) + llm_response = self._query_llm(messages) + + # 5. Parse action from response + thought, action = self._parse_action(llm_response) + + # 6. Execute action + observation = await self._execute_action(action) + + # 7. Build step + step = AgentStep( + step_index=step_index, + screenshot_path=screenshot_path, + action=action, + thought=thought, + observation=observation, + timestamp=time.time(), + url=page_state.get("url", ""), + viewport=page_state.get("viewport"), + coordinates=_extract_coordinates(action), + annotator_instruction=instruction, + ) + + # 8. Emit step event + self._emit_event("step", step.to_dict()) + + return step + + def _build_llm_messages( + self, + screenshot_b64: str, + task_description: str, + instruction: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Build message list for the LLM vision API.""" + messages = [] + + # System message + system_prompt = self.config.system_prompt or DEFAULT_SYSTEM_PROMPT + messages.append({"role": "system", "content": system_prompt}) + + # Task description + task_msg = f"Task: {task_description}" + if instruction: + task_msg += f"\n\nAnnotator instruction: {instruction}" + + # Include recent step history + history_steps = self._steps[-self.config.history_window:] + if history_steps: + history_parts = [] + for s in history_steps: + entry = f"Step {s.step_index}: thought='{s.thought}', action={json.dumps(s.action)}, observation='{s.observation}'" + history_parts.append(entry) + task_msg += "\n\nRecent history:\n" + "\n".join(history_parts) + + messages.append({"role": "user", "content": task_msg}) + + # Current screenshot (as a separate user message with image) + messages.append({ + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": screenshot_b64, + }, + }, + { + "type": "text", + "text": f"Current page screenshot (step {len(self._steps)}). What action should I take next?", + }, + ], + }) + + return messages + + def _init_llm_client(self): + """Initialize the LLM client based on endpoint_type.""" + if self.config.endpoint_type == "anthropic_vision": + try: + import anthropic + except ImportError: + raise RuntimeError( + "anthropic package required. Install with: pip install anthropic" + ) + api_key = self.config.api_key or os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + raise RuntimeError( + "Anthropic API key required. Set in config or ANTHROPIC_API_KEY env var." + ) + self._llm_client = anthropic.Anthropic( + api_key=api_key, timeout=self.config.timeout + ) + elif self.config.endpoint_type == "ollama_vision": + try: + import ollama + except ImportError: + raise RuntimeError( + "ollama package required. Install with: pip install ollama" + ) + host = self.config.base_url or "http://localhost:11434" + self._llm_client = ollama.Client( + host=host, timeout=self.config.timeout + ) + # Verify connectivity + try: + self._llm_client.list() + logger.info(f"Connected to Ollama at {host}, model: {self.config.model}") + except Exception as e: + raise RuntimeError(f"Failed to connect to Ollama at {host}: {e}") + elif self.config.endpoint_type == "openai_vision": + try: + from openai import OpenAI + except ImportError: + raise RuntimeError( + "openai package required. Install with: pip install openai" + ) + base_url = self.config.base_url or "https://api.openai.com/v1" + self._llm_client = OpenAI( + base_url=base_url, + api_key=self.config.api_key or "EMPTY", + timeout=self.config.timeout, + ) + try: + self._llm_client.models.list() + logger.info( + f"Connected to OpenAI-compatible endpoint at {base_url}, " + f"model: {self.config.model}" + ) + except Exception as e: + # Non-fatal: some servers gate /models; the chat call will + # surface a real error if the endpoint is truly unreachable. + logger.warning( + f"Could not list models at {base_url} ({e}); continuing." + ) + else: + raise RuntimeError( + f"Unsupported endpoint_type: {self.config.endpoint_type}. " + f"Supported: 'anthropic_vision', 'ollama_vision', 'openai_vision'." + ) + + def _query_llm(self, messages: List[Dict[str, Any]]) -> str: + """Send messages to the LLM and return the text response.""" + if self.config.endpoint_type == "anthropic_vision": + return self._query_anthropic(messages) + elif self.config.endpoint_type == "ollama_vision": + return self._query_ollama(messages) + elif self.config.endpoint_type == "openai_vision": + return self._query_openai(messages) + raise RuntimeError(f"Unsupported endpoint type: {self.config.endpoint_type}") + + def _query_openai(self, messages: List[Dict[str, Any]]) -> str: + """Query an OpenAI-compatible vision endpoint (OpenAI, vLLM, etc.). + + Converts the internal Anthropic-style message blocks into OpenAI + chat-completions format (image blocks become ``image_url`` data + URIs). Requests a JSON object response when the server supports it, + falling back gracefully if it does not. + """ + oai_messages = [] + for msg in messages: + role = msg["role"] + content = msg.get("content", "") + if isinstance(content, str): + oai_messages.append({"role": role, "content": content}) + continue + parts = [] + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + parts.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image": + src = block.get("source", {}) + if src.get("type") == "base64": + media = src.get("media_type", "image/png") + parts.append({ + "type": "image_url", + "image_url": { + "url": f"data:{media};base64,{src['data']}" + }, + }) + oai_messages.append({"role": role, "content": parts}) + + kwargs = { + "model": self.config.model, + "messages": oai_messages, + "max_tokens": self.config.max_tokens, + "temperature": self.config.temperature, + } + + def _is_rate_limit(exc) -> bool: + if getattr(exc, "status_code", None) == 429: + return True + s = str(exc).lower() + return ("429" in s or "rate limit" in s or "quota" in s + or "resource_exhausted" in s) + + def _create(use_rf: bool): + if use_rf: + return self._llm_client.chat.completions.create( + response_format={"type": "json_object"}, **kwargs) + return self._llm_client.chat.completions.create(**kwargs) + + # Transient 429s (per-minute rate/token bursts) are common mid-run + # even on paid tiers; back off and retry instead of failing the + # whole agent session. + backoffs = [5, 15, 30, 30, 30] + use_rf = True + attempt = 0 + while True: + try: + resp = _create(use_rf) + break + except Exception as e: + if _is_rate_limit(e): + if attempt >= len(backoffs): + raise + wait = backoffs[attempt] + attempt += 1 + logger.warning( + f"[{self.session_id}] LLM 429/rate-limited; " + f"retry {attempt}/{len(backoffs)} in {wait}s" + ) + self._emit_event("thinking", { + "text": f"Rate-limited by the model API; " + f"waiting {wait}s before retryingโ€ฆ" + }) + time.sleep(wait) + continue + if use_rf: + # Server may not support response_format; drop it once. + use_rf = False + continue + raise + return resp.choices[0].message.content or "" + + def _query_anthropic(self, messages: List[Dict[str, Any]]) -> str: + """Query Anthropic Claude with vision support.""" + # Separate system message + system = "" + api_messages = [] + for msg in messages: + if msg["role"] == "system": + system = msg["content"] + else: + api_messages.append(msg) + + kwargs = { + "model": self.config.model, + "max_tokens": self.config.max_tokens, + "temperature": self.config.temperature, + "messages": api_messages, + } + if system: + kwargs["system"] = system + + response = self._llm_client.messages.create(**kwargs) + return response.content[0].text + + def _query_ollama(self, messages: List[Dict[str, Any]]) -> str: + """Query Ollama vision model. + + Converts Anthropic-format messages to Ollama format: + - System messages are prepended to the prompt text + - Multiple user messages are merged into a single message + - Content blocks with images use Ollama's 'images' key + """ + # Extract text and images from Anthropic-format messages + all_text_parts = [] + all_images = [] + for msg in messages: + content = msg.get("content", "") + if msg["role"] == "system": + if isinstance(content, str) and content: + all_text_parts.insert(0, content) + continue + if isinstance(content, list): + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + all_text_parts.append(block["text"]) + elif block.get("type") == "image": + source = block.get("source", {}) + if source.get("type") == "base64": + all_images.append(source["data"]) + elif isinstance(content, str) and content: + all_text_parts.append(content) + + ollama_msg = { + "role": "user", + "content": "\n\n".join(all_text_parts), + } + if all_images: + ollama_msg["images"] = all_images + + options = { + "temperature": self.config.temperature, + "num_predict": self.config.max_tokens, + } + + # Use Ollama's format schema to force structured JSON output + agent_schema = { + "type": "object", + "properties": { + "thought": {"type": "string"}, + "action": { + "type": "object", + "properties": { + "type": {"type": "string"}, + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "text": {"type": "string"}, + "url": {"type": "string"}, + "direction": {"type": "string"}, + "amount": {"type": "integer"}, + "summary": {"type": "string"}, + }, + "required": ["type"], + }, + }, + "required": ["thought", "action"], + } + + response = self._llm_client.chat( + model=self.config.model, + messages=[ollama_msg], + options=options, + format=agent_schema, + ) + + # Extract content from response (handle both dict and Pydantic model) + message = ( + response.get("message") + if hasattr(response, "get") + else getattr(response, "message", None) + ) + if message is None: + raise RuntimeError("No message in Ollama response") + + content = ( + message.get("content") + if hasattr(message, "get") + else getattr(message, "content", None) + ) + + # Some models (e.g. qwen3-vl) put responses in 'thinking' field + # and leave content empty. Extract the agent JSON from thinking. + if not content: + thinking = ( + message.get("thinking") + if hasattr(message, "get") + else getattr(message, "thinking", None) + ) + if thinking: + content = _extract_agent_json(thinking) + + return content or "" + + def _parse_action(self, llm_response: str) -> tuple: + """Parse thought and action from LLM JSON response. + + Returns: + (thought, action_dict) + """ + # Try to extract JSON from response + text = llm_response.strip() + + # Handle markdown code blocks + if "```json" in text: + import re + match = re.search(r"```json\s*([\s\S]*?)\s*```", text) + if match: + text = match.group(1).strip() + elif "```" in text: + import re + match = re.search(r"```\s*([\s\S]*?)\s*```", text) + if match: + text = match.group(1).strip() + + try: + parsed = json.loads(text) + except json.JSONDecodeError: + logger.warning(f"Failed to parse LLM response as JSON: {text[:200]}") + return text, {"type": "wait"} + + thought = parsed.get("thought", "") + action = parsed.get("action", {"type": "wait"}) + + # Validate action has a type + if "type" not in action: + action["type"] = "wait" + + return thought, action + + async def _execute_action(self, action: Dict[str, Any]) -> str: + """Execute an action via Playwright and return observation.""" + action_type = action.get("type", "wait") + pw = self._playwright_session + + try: + if action_type == "click": + x = int(action.get("x", 0)) + y = int(action.get("y", 0)) + success = await pw.click(x, y) + return f"Clicked at ({x}, {y})" if success else f"Click failed at ({x}, {y})" + + elif action_type == "type": + text = action.get("text", "") + # Handle control characters via keyboard.press + if text == "\b": + success = await pw.page.keyboard.press("Backspace") or True + return "Pressed Backspace" + elif text == "\n": + success = await pw.page.keyboard.press("Enter") or True + return "Pressed Enter" + elif text == "\t": + success = await pw.page.keyboard.press("Tab") or True + return "Pressed Tab" + else: + success = await pw.type_text(text) + return f"Typed '{text}'" if success else f"Type failed: '{text}'" + + elif action_type == "scroll": + direction = action.get("direction", "down") + amount = int(action.get("amount", 300)) + dy = amount if direction == "down" else -amount + success = await pw.scroll(0, dy) + return f"Scrolled {direction} by {amount}px" if success else "Scroll failed" + + elif action_type == "navigate": + url = action.get("url", "") + success = await pw.navigate(url) + return f"Navigated to {url}" if success else f"Navigation failed: {url}" + + elif action_type == "wait": + await asyncio.sleep(1) + return "Waited 1 second" + + elif action_type == "done": + summary = action.get("summary", "Task completed") + return summary + + else: + logger.warning(f"Unknown action type: {action_type}") + return f"Unknown action: {action_type}" + + except Exception as e: + logger.error(f"Action execution error: {e}") + return f"Error executing {action_type}: {e}" + + async def _process_takeover_actions(self): + """Process manual actions submitted during takeover mode.""" + try: + action = self._takeover_actions.get_nowait() + except Empty: + return + + pw = self._playwright_session + if not pw: + return + + observation = await self._execute_action(action) + + # Take screenshot after manual action + screenshot_bytes = await pw.screenshot() + step_index = len(self._steps) + screenshot_path = os.path.join( + self.screenshot_dir, f"step_{step_index:03d}_manual.png" + ) + if screenshot_bytes: + with open(screenshot_path, "wb") as f: + f.write(screenshot_bytes) + + page_state = await pw.get_state() + + step = AgentStep( + step_index=step_index, + screenshot_path=screenshot_path, + action={**action, "_manual": True}, + thought="[Manual takeover action]", + observation=observation, + timestamp=time.time(), + url=page_state.get("url", ""), + viewport=page_state.get("viewport"), + coordinates=_extract_coordinates(action), + ) + self._steps.append(step) + self._emit_event("step", step.to_dict()) + + # --- Trace export --- + + def get_trace(self) -> Dict[str, Any]: + """Export the session as a web_agent_trace-compatible dict.""" + return { + "steps": [s.to_dict() for s in self._steps], + "task_description": "", # Set by caller + "session_id": self.session_id, + "agent_config": { + "model": self.config.model, + "endpoint_type": self.config.endpoint_type, + "max_steps": self.config.max_steps, + }, + "annotator_interactions": self._interactions, + "state": self.state.value, + "total_steps": len(self._steps), + } + + def get_state_summary(self) -> Dict[str, Any]: + """Get a summary of current state for API responses.""" + return { + "session_id": self.session_id, + "state": self.state.value, + "step_count": len(self._steps), + "error": self._error, + "has_instructions_pending": not self._instruction_queue.empty(), + } + + +def _extract_agent_json(text: str) -> str: + """Extract the last valid JSON object containing 'thought' or 'action' from text. + + Some models (qwen3-vl) put their chain-of-thought in the thinking field + with the actual JSON answer embedded in the text. This function finds + that JSON, skipping any example/template JSON from the prompt. + """ + import re + + # Find all JSON-like blocks (balanced braces) + candidates = [] + depth = 0 + start = None + for i, ch in enumerate(text): + if ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0 and start is not None: + candidates.append(text[start : i + 1]) + start = None + + # Try each candidate (last first โ€” most likely to be the final answer) + for candidate in reversed(candidates): + try: + parsed = json.loads(candidate) + if isinstance(parsed, dict) and ("thought" in parsed or "action" in parsed): + return candidate + except (json.JSONDecodeError, ValueError): + continue + + # Fallback: try greedy regex for any JSON + match = re.search(r"\{[^{}]*\}", text) + return match.group(0) if match else "" + + +def _extract_coordinates(action: Dict[str, Any]) -> Optional[Dict[str, int]]: + """Extract x, y coordinates from an action if present.""" + if "x" in action and "y" in action: + return {"x": int(action["x"]), "y": int(action["y"])} + return None diff --git a/potato/agent_runner_manager.py b/potato/agent_runner_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b1903506951ab6c4ad2bf06166b96c9dfdad517e --- /dev/null +++ b/potato/agent_runner_manager.py @@ -0,0 +1,226 @@ +""" +Agent Runner Session Manager + +Singleton that manages active AgentRunner sessions. +Keyed by "{user_id}:{instance_id}" for per-user, per-instance isolation. +Includes TTL-based cleanup and max concurrent session limits. +""" + +import atexit +import logging +import threading +import time +from typing import Dict, Optional + +from potato.agent_runner import AgentConfig, AgentRunner, AgentState + +logger = logging.getLogger(__name__) + +# Default limits +DEFAULT_MAX_SESSIONS = 10 +DEFAULT_SESSION_TTL = 3600 # 1 hour + + +class AgentRunnerManager: + """ + Manages active AgentRunner sessions with lifecycle control. + + Thread-safe singleton. Sessions are keyed by "{user_id}:{instance_id}". + """ + + _instance = None + _lock = threading.Lock() + + def __init__( + self, + max_sessions: int = DEFAULT_MAX_SESSIONS, + session_ttl: int = DEFAULT_SESSION_TTL, + ): + self._sessions: Dict[str, AgentRunner] = {} + self._session_created: Dict[str, float] = {} + self._session_meta: Dict[str, Dict] = {} + self._lock = threading.Lock() + self.max_sessions = max_sessions + self.session_ttl = session_ttl + + # Start cleanup thread + self._cleanup_stop = threading.Event() + self._cleanup_thread = threading.Thread( + target=self._cleanup_loop, daemon=True, name="agent-cleanup" + ) + self._cleanup_thread.start() + + @classmethod + def get_instance(cls, **kwargs) -> "AgentRunnerManager": + """Get or create the singleton instance.""" + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = cls(**kwargs) + return cls._instance + + @classmethod + def clear_instance(cls): + """Clear the singleton (for testing).""" + with cls._lock: + if cls._instance is not None: + cls._instance.shutdown() + cls._instance = None + + def create_session( + self, + user_id: str, + instance_id: str, + config: AgentConfig, + screenshot_dir: str, + ) -> AgentRunner: + """ + Create a new agent session. + + Args: + user_id: Annotator user ID + instance_id: Annotation instance ID + config: Agent configuration + screenshot_dir: Directory to store screenshots + + Returns: + AgentRunner instance + + Raises: + RuntimeError: If max sessions reached or session already exists + """ + session_key = f"{user_id}:{instance_id}" + + with self._lock: + # Clean up expired sessions first + self._cleanup_expired_locked() + + # Check for existing active session + if session_key in self._sessions: + existing = self._sessions[session_key] + if existing.state in (AgentState.RUNNING, AgentState.PAUSED, AgentState.TAKEOVER): + raise RuntimeError( + f"Active session already exists for {session_key}. " + f"Stop it first." + ) + # Old completed/error session โ€” remove it + del self._sessions[session_key] + del self._session_created[session_key] + if session_key in self._session_meta: + del self._session_meta[session_key] + + # Check capacity + active_count = sum( + 1 + for s in self._sessions.values() + if s.state in (AgentState.RUNNING, AgentState.PAUSED, AgentState.TAKEOVER) + ) + if active_count >= self.max_sessions: + raise RuntimeError( + f"Maximum concurrent sessions ({self.max_sessions}) reached" + ) + + import uuid + session_id = str(uuid.uuid4())[:12] + runner = AgentRunner(session_id, config, screenshot_dir) + + self._sessions[session_key] = runner + self._session_created[session_key] = time.time() + self._session_meta[session_key] = { + "user_id": user_id, + "instance_id": instance_id, + "session_id": session_id, + } + + logger.info( + f"Created agent session {session_id} for {session_key}" + ) + return runner + + def get_session(self, session_id: str) -> Optional[AgentRunner]: + """Get a session by its session_id.""" + with self._lock: + for runner in self._sessions.values(): + if runner.session_id == session_id: + return runner + return None + + def get_session_by_key(self, user_id: str, instance_id: str) -> Optional[AgentRunner]: + """Get a session by user_id and instance_id.""" + session_key = f"{user_id}:{instance_id}" + with self._lock: + return self._sessions.get(session_key) + + def remove_session(self, session_id: str): + """Remove a session by session_id.""" + with self._lock: + key_to_remove = None + for key, runner in self._sessions.items(): + if runner.session_id == session_id: + key_to_remove = key + break + if key_to_remove: + runner = self._sessions.pop(key_to_remove) + self._session_created.pop(key_to_remove, None) + self._session_meta.pop(key_to_remove, None) + runner.stop() + logger.info(f"Removed agent session {session_id}") + + def list_sessions(self) -> list: + """List all active sessions.""" + with self._lock: + result = [] + for key, runner in self._sessions.items(): + meta = self._session_meta.get(key, {}) + result.append({ + "session_id": runner.session_id, + "user_id": meta.get("user_id"), + "instance_id": meta.get("instance_id"), + "state": runner.state.value, + "step_count": runner.step_count, + "created": self._session_created.get(key), + }) + return result + + def _cleanup_expired_locked(self): + """Remove expired sessions. Must be called with self._lock held.""" + now = time.time() + expired_keys = [] + for key, created_at in self._session_created.items(): + if now - created_at > self.session_ttl: + runner = self._sessions.get(key) + if runner and runner.state in (AgentState.COMPLETED, AgentState.ERROR, AgentState.IDLE): + expired_keys.append(key) + elif runner and now - created_at > self.session_ttl * 2: + # Force-stop sessions that have been running too long + runner.stop() + expired_keys.append(key) + + for key in expired_keys: + self._sessions.pop(key, None) + self._session_created.pop(key, None) + self._session_meta.pop(key, None) + logger.info(f"Cleaned up expired session: {key}") + + def _cleanup_loop(self): + """Background cleanup thread.""" + while not self._cleanup_stop.is_set(): + self._cleanup_stop.wait(60) # Check every 60 seconds + if self._cleanup_stop.is_set(): + break + with self._lock: + self._cleanup_expired_locked() + + def shutdown(self): + """Stop all sessions and cleanup thread.""" + self._cleanup_stop.set() + with self._lock: + for key, runner in self._sessions.items(): + try: + runner.stop() + except Exception as e: + logger.warning(f"Error stopping session {key}: {e}") + self._sessions.clear() + self._session_created.clear() + self._session_meta.clear() + logger.info("AgentRunnerManager shut down") diff --git a/potato/agreement.py b/potato/agreement.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcf7108dcebbf64caf6fc58954528e6155c8053 --- /dev/null +++ b/potato/agreement.py @@ -0,0 +1,278 @@ +""" +Inter-Annotator Agreement Calculation Module + +This module provides functionality for calculating inter-annotator agreement metrics, +including Krippendorff's alpha, Cohen's kappa (pairwise), and Fleiss' kappa +(N raters), from annotation data. It supports both rating agreement (interval +metric) and skip agreement (nominal metric) calculations. + +The module processes annotation files in JSON format and outputs agreement statistics +along with a CSV file containing the processed annotation data. +""" + +import argparse +from itertools import combinations +import simpledorff +from simpledorff.metrics import * +import ujson +import pandas as pd + +from collections import defaultdict +import numpy as np + + +def get_nans(shape): + """ + Create a numpy array filled with NaN values. + + Args: + shape: The shape of the array to create + + Returns: + numpy.ndarray: Array filled with NaN values + """ + ar = np.empty(shape) + ar[:] = np.NaN + return ar + + +def cohen_kappa_pairwise(reliability_df): + """ + Compute Cohen's kappa for every pair of annotators and return aggregate stats. + + Cohen's kappa is defined for exactly two raters. With N>2 raters we compute + kappa for each pair on the items they both rated, then return the mean and the + per-pair breakdown. Pairs that share fewer than 2 items are skipped. + + Args: + reliability_df: long-format DataFrame with columns + unit (item id), annotator (user), annotation (label value). + + Returns: + dict with keys: mean_kappa (float | None), pairs (list of + {annotator_a, annotator_b, kappa, n_items}), n_pairs_evaluated, + n_pairs_skipped. + """ + from sklearn.metrics import cohen_kappa_score + + annotators = sorted(reliability_df["annotator"].unique()) + pairs = [] + skipped = 0 + + for a, b in combinations(annotators, 2): + a_rows = reliability_df[reliability_df["annotator"] == a].set_index("unit")["annotation"] + b_rows = reliability_df[reliability_df["annotator"] == b].set_index("unit")["annotation"] + shared = a_rows.index.intersection(b_rows.index) + if len(shared) < 2: + skipped += 1 + continue + + y_a = a_rows.loc[shared].astype(str).tolist() + y_b = b_rows.loc[shared].astype(str).tolist() + try: + kappa = float(cohen_kappa_score(y_a, y_b)) + except Exception: + skipped += 1 + continue + pairs.append({ + "annotator_a": a, + "annotator_b": b, + "kappa": round(kappa, 4), + "n_items": int(len(shared)), + }) + + mean_kappa = (sum(p["kappa"] for p in pairs) / len(pairs)) if pairs else None + return { + "mean_kappa": round(mean_kappa, 4) if mean_kappa is not None else None, + "pairs": pairs, + "n_pairs_evaluated": len(pairs), + "n_pairs_skipped": skipped, + } + + +def fleiss_kappa(reliability_df): + """ + Compute Fleiss' kappa for N raters over a categorical label set. + + Fleiss' kappa assumes the same number of ratings per item but tolerates + different rater identities per item. Items with fewer than 2 ratings are + dropped; the remaining items are padded by repeating their available + ratings up to the per-item rater count (`n_raters = max ratings per item`). + When per-item rater counts vary widely the metric is approximate; we report + `n_raters` and `n_items_evaluated` so the caller can judge. + + Args: + reliability_df: long-format DataFrame with columns + unit (item id), annotator (user), annotation (label value). + + Returns: + dict with keys: kappa (float | None), n_items_evaluated (int), + n_raters (int), n_categories (int), interpretation (str). + """ + if reliability_df.empty: + return {"kappa": None, "n_items_evaluated": 0, "n_raters": 0, + "n_categories": 0, "interpretation": "No data"} + + df = reliability_df.copy() + df["annotation"] = df["annotation"].astype(str) + + counts_by_item = df.groupby(["unit", "annotation"]).size().unstack(fill_value=0) + items_with_ratings = counts_by_item.sum(axis=1) + counts_by_item = counts_by_item.loc[items_with_ratings >= 2] + + if counts_by_item.empty: + return {"kappa": None, "n_items_evaluated": 0, "n_raters": 0, + "n_categories": int(df["annotation"].nunique()), + "interpretation": "No items with >=2 raters"} + + n_raters = int(counts_by_item.sum(axis=1).max()) + n_items = int(counts_by_item.shape[0]) + n_categories = int(counts_by_item.shape[1]) + + matrix = counts_by_item.to_numpy(dtype=float) + row_sums = matrix.sum(axis=1, keepdims=True) + row_sums[row_sums == 0] = 1.0 + matrix = matrix * (n_raters / row_sums) + + p_j = matrix.sum(axis=0) / (n_items * n_raters) + if n_raters < 2: + return {"kappa": None, "n_items_evaluated": n_items, "n_raters": n_raters, + "n_categories": n_categories, + "interpretation": "Need >=2 raters per item"} + p_i = (np.sum(matrix ** 2, axis=1) - n_raters) / (n_raters * (n_raters - 1)) + p_bar = float(p_i.mean()) + p_e = float(np.sum(p_j ** 2)) + + if p_e >= 1.0: + kappa = 1.0 if p_bar >= 1.0 else 0.0 + else: + kappa = (p_bar - p_e) / (1 - p_e) + + return { + "kappa": round(float(kappa), 4), + "n_items_evaluated": n_items, + "n_raters": n_raters, + "n_categories": n_categories, + "interpretation": interpret_kappa(kappa), + } + + +def interpret_kappa(kappa): + """Landis & Koch (1977) interpretation bands for kappa-family metrics.""" + if kappa is None: + return "No agreement computable" + if kappa < 0: + return "Worse than chance" + if kappa < 0.21: + return "Slight" + if kappa < 0.41: + return "Fair" + if kappa < 0.61: + return "Moderate" + if kappa < 0.81: + return "Substantial" + return "Almost perfect" + + +def flatten(annotations): + """ + Flatten annotation data structure for processing. + + Converts a list of annotation dictionaries into a format where each + annotation is a dictionary mapping user IDs to their labels. + + Args: + annotations: List of annotation dictionaries + + Returns: + list: Flattened annotation data structure + + Example: + Input: [{"user": "user1", "label": "positive"}, {"user": "user2", "label": "negative"}] + Output: [{"user1": "positive", "user2": "negative"}] + """ + return [{a["user"]: a["label"] for a in ann} for ann in annotations] + + +def main(args): + """ + Main function for calculating inter-annotator agreement. + + This function processes annotation data from a JSON file, calculates + Krippendorff's alpha for both rating agreement and skip agreement, + and outputs the results along with a CSV file of the processed data. + + Args: + args: Command line arguments containing file paths + + Side Effects: + - Reads annotation data from input file + - Prints agreement statistics to console + - Writes processed data to output CSV file + + The function processes the first 385 annotations by default and handles + missing annotations and skipped items appropriately. + """ + # Load annotation data from JSON file + with open(args.file, "r") as f: + annotations = [ujson.loads(line)["annotations"] for line in f] + + # Extract unique user IDs from all annotations + users = set([a["user"] for ann in annotations for a in ann]) + annotations = flatten(annotations) + + # Limit to first 385 annotations (configurable limit) + annotations = annotations[:385] + + # Create data matrix for agreement calculation + # Each row represents a user, each column represents an annotation + # -1 values indicate skipped annotations, NaN indicates missing annotations + data = [ + [np.nan if user not in a or int(a[user]) == -1 else int(a[user]) for a in annotations] + for user in users + ] + + # Create skip data matrix (boolean indicating if annotation was skipped) + skip_data = [ + [np.nan if user not in a else int(a[user]) < 0 for a in annotations] for user in users + ] + + # Calculate statistics for each user + labeled = ~np.isnan(data) + skipped = [ + [False if user not in a else int(a[user]) < 0 for a in annotations] for user in users + ] + + # Print summary statistics + print("calculating over:") + for user, skip in zip(labeled, skipped): + print("labeled:", sum(user)) + print("skipped:", sum(skip)) + + # Count instances where all users provided annotations + print(np.all(labeled, axis=0).sum()) + + # Calculate and print Krippendorff's alpha for rating agreement + # Uses interval metric for continuous rating scales + print("rating agreement:") + print(simpledorff.calculate_krippendorffs_alpha(pd.DataFrame(data),metric_fn=interval_metric)) + + # Calculate and print Krippendorff's alpha for skip agreement + # Uses nominal metric for binary skip/no-skip decisions + print("skip agreement:") + print(simpledorff.calculate_krippendorffs_alpha(pd.DataFrame(data),metric_fn=nominal_metric)) + + # Write processed data to CSV file + with open(args.outfile, "w") as f: + for row in zip(*data): + f.write(",".join([str(a) for a in row]) + "\n") + + +if __name__ == "__main__": + # Set up command line argument parsing + parser = argparse.ArgumentParser( + description="Calculate Krippendorf's alpha from given JSON file of annotations" + ) + parser.add_argument("file", help="path to JSON file") + parser.add_argument("outfile", help="write path to CSV") + main(parser.parse_args()) diff --git a/potato/ai/__init__.py b/potato/ai/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2f15af8c8a1ce1da66103652416ec60c2924fb47 --- /dev/null +++ b/potato/ai/__init__.py @@ -0,0 +1 @@ +from .ai_help_wrapper import generate_ai_help_html \ No newline at end of file diff --git a/potato/ai/ai_cache.py b/potato/ai/ai_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..2c3222feb43d1b4755d8dfd89626480364705f6f --- /dev/null +++ b/potato/ai/ai_cache.py @@ -0,0 +1,1473 @@ +from __future__ import annotations +import json +import logging +import os +from typing import Any, Dict, Union +import requests +from tqdm import tqdm +import time +from concurrent.futures import ThreadPoolExecutor +import threading +from builtins import open +from potato.server_utils.config_module import config + +logger = logging.getLogger(__name__) + +from potato.item_state_management import get_item_state_manager +from potato.ai.ai_endpoint import ( + AIEndpointFactory, + Annotation_Type, + AnnotationInput, + ImageData, + VisualAnnotationInput, + ModelCapabilities, +) +from potato.ai.ollama_endpoint import OllamaEndpoint +from potato.ai.openrouter_endpoint import OpenRouterEndpoint +from potato.ai.ai_prompt import ModelManager, get_ai_prompt + + +AICACHEMANAGER = None + + +def _get_scheme_field(annotation_id: int, field: str, default=None): + """Safely get a field from an annotation scheme with a clear error message.""" + schemes = config.get("annotation_schemes", []) + if annotation_id >= len(schemes): + raise ValueError( + f"AI cache: annotation_id {annotation_id} out of range " + f"(only {len(schemes)} scheme(s) configured)" + ) + scheme = schemes[annotation_id] + if default is not None: + return scheme.get(field, default) + if field not in scheme: + scheme_name = scheme.get("name", f"index {annotation_id}") + scheme_type = scheme.get("annotation_type", "unknown") + raise ValueError( + f"AI cache: annotation scheme '{scheme_name}' (type '{scheme_type}') " + f"missing required field '{field}'" + ) + return scheme[field] + + +def _get_instance_text(instance_id: int) -> str: + """Get the text content from an instance using the configured text_key.""" + item = get_item_state_manager().items()[instance_id] + item_data = item.get_data() + + # Get the configured text_key + text_key = config.get("item_properties", {}).get("text_key", "text") + + # Try the configured text_key first + if text_key in item_data: + return item_data[text_key] + + # Fall back to common keys + for key in ['text', 'content', 'message']: + if key in item_data: + return item_data[key] + + # Last resort: return any string value + for value in item_data.values(): + if isinstance(value, str): + return value + + return str(item_data) + +def _is_image_url(text: str) -> bool: + """Check if text appears to be an image URL.""" + if not isinstance(text, str): + return False + text_lower = text.lower() + # Check for image extensions + image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'] + if any(ext in text_lower for ext in image_extensions): + return True + # Check for common image hosting services + image_hosts = ['unsplash.com', 'imgur.com', 'flickr.com', 'picsum.photos'] + if any(host in text_lower for host in image_hosts): + return True + # Check if URL starts with http and might be an image + if text_lower.startswith(('http://', 'https://')) and 'image' in text_lower: + return True + return False + +def _get_image_data_from_url(url: str) -> ImageData: + """Download image from URL and return as ImageData. + + Includes SSRF protection to prevent fetching from private/internal IPs. + """ + import base64 + import ipaddress + import socket + from urllib.parse import urlparse + + # SSRF protection: validate URL scheme and resolve hostname + try: + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https'): + logger.warning(f"Blocked non-HTTP image URL: {url[:100]}") + return None + + hostname = parsed.hostname + if hostname: + addr_info = socket.getaddrinfo(hostname, None) + for info in addr_info: + ip_str = info[4][0] + try: + ip = ipaddress.ip_address(ip_str) + if ip.is_private or ip.is_loopback or ip.is_link_local: + logger.warning( + f"Blocked image URL resolving to private IP: " + f"{hostname} -> {ip_str}" + ) + return None + except ValueError: + pass + except Exception as e: + logger.warning(f"Failed to validate image URL {url[:100]}: {e}") + return None + + try: + response = requests.get(url, timeout=30) + response.raise_for_status() + b64_data = base64.b64encode(response.content).decode('utf-8') + # Determine mime type from content-type header or URL + content_type = response.headers.get('content-type', 'image/jpeg') + return ImageData(source='base64', data=b64_data, mime_type=content_type) + except Exception as e: + logger.error(f"Failed to download image from {url}: {e}") + return None + +def init_ai_cache_manager(): + global AICACHEMANAGER + if AICACHEMANAGER is None: + AICACHEMANAGER = AiCacheManager() + + return AICACHEMANAGER + +def get_ai_cache_manager(): + """Get the AI cache manager instance. Returns None if not initialized (AI support disabled).""" + global AICACHEMANAGER + return AICACHEMANAGER + +def clear_ai_cache_manager(): + """Clear the AI cache manager singleton. Used for testing.""" + global AICACHEMANAGER + AICACHEMANAGER = None + +class AiCacheManager: + def __init__(self): + ai_support = config["ai_support"] + if not ai_support["enabled"]: + return + cache_config = ai_support.get("cache_config", {}) + ai_config = ai_support.get("ai_config", {}) + include = ai_config.get("include") or {} + special_include = include.get("special_include", None) + self.include_all = include.get("all", False) + self.special_includes = {} + + self.model_manager = ModelManager() + self.model_manager.load_models_module() + + if special_include: + for page_key, page_value in special_include.items(): + # Convert string keys to integers for easier lookup + page_index = int(page_key) + self.special_includes[page_index] = {} + for annotation_id, annotation_types in page_value.items(): + annotation_id_int = int(annotation_id) + self.special_includes[page_index][annotation_id_int] = annotation_types + + # Disk cache configuration. + # F-028: tolerate a partial/absent ai_cache config (e.g. AI support + # enabled for ICL with no disk_cache block) instead of crashing boot + # with KeyError: 'disk_cache'. + disk_cache_cfg = cache_config.get("disk_cache", {}) if isinstance(cache_config, dict) else {} + self.disk_cache_enabled = disk_cache_cfg.get("enabled", False) + + disk_cache_path = disk_cache_cfg.get("path") + if self.disk_cache_enabled and not disk_cache_path: + raise Exception("You have enable disk cache, but you did not specific the path!") + self.disk_persistence_path = disk_cache_path + + # Validate cache path stays within task directory + if self.disk_persistence_path: + task_dir = os.path.abspath(config.get("task_dir", ".")) + cache_abs = os.path.abspath( + os.path.join(task_dir, self.disk_persistence_path) + if not os.path.isabs(self.disk_persistence_path) + else self.disk_persistence_path + ) + if not cache_abs.startswith(task_dir + os.sep) and cache_abs != task_dir: + raise ValueError( + f"Cache path '{self.disk_persistence_path}' resolves to " + f"'{cache_abs}' which is outside the task directory " + f"'{task_dir}'. Path traversal is not allowed." + ) + + # Prefetch configuration โ€” clamp to sane ranges. + # F-028: default to no prefetch when the prefetch block is absent + # (e.g. cache_config: {enabled: false}) instead of KeyError on boot. + prefetch_cfg = cache_config.get("prefetch", {}) if isinstance(cache_config, dict) else {} + self.warm_up_page_count = max(0, min(int(prefetch_cfg.get("warm_up_page_count", 0)), 10000)) + self.prefetch_page_count_on_next = max(0, min(int(prefetch_cfg.get("on_next", 0)), 10000)) + self.prefetch_page_count_on_prev = max(0, min(int(prefetch_cfg.get("on_prev", 0)), 10000)) + + # Option highlighting configuration + option_highlighting = ai_support.get("option_highlighting", {}) + self.option_highlighting_enabled = option_highlighting.get("enabled", False) + self.option_highlighting_top_k = option_highlighting.get("top_k", 3) + self.option_highlighting_dim_opacity = option_highlighting.get("dim_opacity", 0.4) + self.option_highlighting_auto_apply = option_highlighting.get("auto_apply", True) + self.option_highlighting_schemas = option_highlighting.get("schemas", None) # None means all + # Prefetch count for option highlighting โ€” clamp to sane range + self.option_highlighting_prefetch_count = max(0, min( + int(option_highlighting.get("prefetch_count", 20)), 10000 + )) + + # Threading + self.in_progress = {} + self.lock = threading.RLock() + self.executor = ThreadPoolExecutor(max_workers=20) + + AIEndpointFactory.register_endpoint("ollama", OllamaEndpoint) + AIEndpointFactory.register_endpoint("open_router", OpenRouterEndpoint) + + # Register visual AI endpoints + try: + from potato.ai.yolo_endpoint import YOLOEndpoint + AIEndpointFactory.register_endpoint("yolo", YOLOEndpoint) + except ImportError: + logger.debug("YOLO endpoint not available (ultralytics not installed)") + + try: + from potato.ai.ollama_vision_endpoint import OllamaVisionEndpoint + AIEndpointFactory.register_endpoint("ollama_vision", OllamaVisionEndpoint) + except ImportError: + logger.debug("Ollama Vision endpoint not available") + + try: + from potato.ai.openai_vision_endpoint import OpenAIVisionEndpoint + AIEndpointFactory.register_endpoint("openai_vision", OpenAIVisionEndpoint) + except ImportError: + logger.debug("OpenAI Vision endpoint not available") + + try: + from potato.ai.anthropic_vision_endpoint import AnthropicVisionEndpoint + AIEndpointFactory.register_endpoint("anthropic_vision", AnthropicVisionEndpoint) + except ImportError: + logger.debug("Anthropic Vision endpoint not available") + + # Degrade gracefully if the AI backend (e.g. a local Ollama/vLLM server) + # is unreachable at boot: log a warning and serve the task with AI + # support disabled rather than aborting server startup. + try: + self.ai_endpoint = AIEndpointFactory.create_endpoint(config) + except Exception as e: + logger.warning( + "AI endpoint unavailable at startup (%s). Continuing with AI " + "support disabled. Check that your AI backend is running.", e + ) + self.ai_endpoint = None + + # Create visual endpoint if different from main endpoint + self.visual_endpoint = None + visual_endpoint_type = config.get("ai_support", {}).get("visual_endpoint_type") + if visual_endpoint_type and visual_endpoint_type != config.get("ai_support", {}).get("endpoint_type"): + visual_config = { + "ai_support": { + "enabled": True, + "endpoint_type": visual_endpoint_type, + "ai_config": config.get("ai_support", {}).get("visual_ai_config", config.get("ai_support", {}).get("ai_config", {})) + } + } + try: + self.visual_endpoint = AIEndpointFactory.create_endpoint(visual_config) + except Exception as e: + logger.warning( + "Visual AI endpoint unavailable at startup (%s). Continuing " + "without visual AI support.", e + ) + self.visual_endpoint = None + + annotation_scheme = config.get("annotation_schemes") + self.annotations = [] + for scheme in annotation_scheme: + self.annotations.append(scheme) + + # Check if main endpoint supports vision + self.endpoint_supports_vision = hasattr(self.ai_endpoint, 'query_with_image') + logger.info(f"AI endpoint supports vision: {self.endpoint_supports_vision}") + + # Initialize cache + if self.disk_cache_enabled: + self.load_cache_from_disk() + self.start_warmup() + + def _validate_assistant_compatibility( + self, instance_id: int, annotation_id: int, ai_assistant: str + ) -> tuple: + """ + Validate that the AI assistant is compatible with the input type and model capabilities. + + Args: + instance_id: The instance/item index + annotation_id: The annotation scheme index + ai_assistant: Type of assistance ('hint', 'keyword', 'rationale', 'detection', etc.) + + Returns: + Tuple of (is_valid: bool, error_message: str) + If valid, error_message is empty string. + """ + try: + text = _get_instance_text(instance_id) + is_image = _is_image_url(text) + + # Determine which endpoint to use + if is_image and self.visual_endpoint: + endpoint = self.visual_endpoint + elif is_image and self.endpoint_supports_vision: + endpoint = self.ai_endpoint + else: + endpoint = self.ai_endpoint + + # Get capabilities from endpoint + capabilities = getattr(endpoint, 'CAPABILITIES', None) + + if capabilities is None: + # No capabilities declared - allow all (backward compatibility) + logger.debug(f"Endpoint {type(endpoint).__name__} has no CAPABILITIES, allowing {ai_assistant}") + return True, "" + + # Check if the assistant type is supported + if not capabilities.supports_assistant(ai_assistant, is_image): + input_type = "image" if is_image else "text" + return False, ( + f"Model {type(endpoint).__name__} does not support '{ai_assistant}' " + f"for {input_type} content" + ) + + return True, "" + + except Exception as e: + logger.warning(f"Error validating assistant compatibility: {e}") + # On validation error, allow the request (fail open for now) + return True, "" + + def get_endpoint_capabilities(self, for_image: bool = False) -> ModelCapabilities: + """ + Get the capabilities of the appropriate endpoint for the given input type. + + Args: + for_image: Whether the input is an image + + Returns: + ModelCapabilities instance, or a default permissive one if not declared + """ + if for_image and self.visual_endpoint: + endpoint = self.visual_endpoint + elif for_image and self.endpoint_supports_vision: + endpoint = self.ai_endpoint + else: + endpoint = self.ai_endpoint + + capabilities = getattr(endpoint, 'CAPABILITIES', None) + if capabilities is None: + # Return permissive defaults for backward compatibility + return ModelCapabilities( + text_generation=True, + vision_input=for_image, + bounding_box_output=False, + text_classification=True, + image_classification=for_image, + rationale_generation=True, + keyword_extraction=not for_image, + ) + return capabilities + + def _get_ai_with_vision_support(self, text: str, prompt: str, output_format) -> str: + """ + Get AI response, using vision if text is an image URL and endpoint supports it. + """ + # Check if we should use vision + if self.endpoint_supports_vision and _is_image_url(text): + logger.debug(f"Using vision query for image URL: {text[:50]}...") + image_data = _get_image_data_from_url(text) + if image_data: + try: + return self.ai_endpoint.query_with_image(prompt, image_data, output_format) + except Exception as e: + logger.error(f"Vision query failed: {e}") + # Fall back to text query + + # Fall back to regular text query + return self.ai_endpoint.query(prompt, output_format) + + def start_warmup(self): + self.start_prefetch(0, self.warm_up_page_count) + + # Also prefetch option highlights if enabled + if self.option_highlighting_enabled: + self.start_option_highlight_prefetch(0, self.warm_up_page_count) + + total = len(self.in_progress) + desc = "Preloading the AI" + + progress_bar = tqdm(total=total, desc=desc, unit="item") + + def count_completed(): + return total - len(self.in_progress) + + prev_done = 0 + while self.in_progress: + current_done = count_completed() + progress_bar.update(current_done - prev_done) + prev_done = current_done + time.sleep(0.2) + + final_done = count_completed() + if final_done > prev_done: + progress_bar.update(final_done - prev_done) + + progress_bar.close() + + def load_disk_cache_data(self, file_path: str) -> Dict[str, Any]: + """loads the cache JSON from disk and returns a dictionary of stringified keys to values.""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + logger.error(f"Error loading disk cache: {e}") + return {} + + def load_cache_from_disk(self): + """Initializes disk cache file if it doesn't exist.""" + if not self.disk_cache_enabled or not self.disk_persistence_path: + return + + if os.path.exists(self.disk_persistence_path): + data = self.load_disk_cache_data(self.disk_persistence_path) + logger.info(f"Disk cache initialized with {len(data)} items") + else: + try: + # Create parent directory if it doesn't exist + os.makedirs(os.path.dirname(self.disk_persistence_path), exist_ok=True) + with open(self.disk_persistence_path, 'w', encoding='utf-8') as file: + json.dump({}, file) + logger.info(f"Initialized empty disk cache at {self.disk_persistence_path}") + except Exception as e: + logger.error(f"Failed to create disk cache: {e}") + + def save_cache_to_disk(self, key, value): + """saves a single key-value pair to disk cache using atomic write.""" + if not self.disk_cache_enabled or not self.disk_persistence_path: + return + + try: + os.makedirs(os.path.dirname(self.disk_persistence_path), exist_ok=True) + + # Load existing disk data first + existing_disk_data = {} + if os.path.exists(self.disk_persistence_path): + existing_disk_data = self.load_disk_cache_data(self.disk_persistence_path) + + # Add the new key-value pair + existing_disk_data[str(key)] = value + + temp_path = self.disk_persistence_path + ".tmp" + with open(temp_path, 'w', encoding='utf-8') as f: + json.dump(existing_disk_data, f, indent=2, ensure_ascii=False) + os.rename(temp_path, self.disk_persistence_path) + except Exception as e: + logger.error(f"Error saving cache to disk: {e}") + + def add_to_cache(self, key, value): + """inserts a key-value into the disk cache.""" + with self.lock: + if self.disk_cache_enabled: + self.save_cache_to_disk(key, value) + + def get_from_cache(self, key): + """Tries to retrieve the item from disk cache.""" + with self.lock: + # Try disk cache + if self.disk_cache_enabled and self.disk_persistence_path and os.path.exists(self.disk_persistence_path): + try: + disk_data = self.load_disk_cache_data(self.disk_persistence_path) + key_str = str(key) + if key_str in disk_data: + return disk_data[key_str] + except Exception as e: + logger.error(f"Error reading from disk: {e}") + return None + + def generate_likert(self, instance_id: int, annotation_id: int, ai_assistant: str) -> str: + from string import Template + annotation_type = _get_scheme_field(annotation_id, "annotation_type") + description = _get_scheme_field(annotation_id, "description") + text = _get_instance_text(instance_id) + min_label = _get_scheme_field(annotation_id, "min_label") + max_label = _get_scheme_field(annotation_id, "max_label") + size = _get_scheme_field(annotation_id, "size") + + ai_prompt = get_ai_prompt() + output_format = self.model_manager.get_model_class_by_name(ai_prompt[annotation_type].get(ai_assistant).get("output_format")) + + # Check if we should use vision endpoint for image-based content + if self.endpoint_supports_vision and _is_image_url(text): + logger.debug(f"Using vision for likert {ai_assistant} on image: {text[:50]}...") + image_data = _get_image_data_from_url(text) + if image_data: + # Build vision-specific prompts based on ai_assistant type + if ai_assistant == "hint": + prompt = f"""Look at this image and help with the following annotation task: + +Task: {description} +Rating scale: {size} points, from "{min_label}" (1) to "{max_label}" ({size}) + +Please analyze the image and suggest an appropriate rating with a brief explanation. +Respond in JSON format: {{"hint": "", "suggestive_choice": ""}}""" + elif ai_assistant == "rationale": + prompt = f"""Look at this image and explain the reasoning for different rating choices: + +Task: {description} +Rating scale: {size} points, from "{min_label}" (1) to "{max_label}" ({size}) + +For each possible rating, explain what visual evidence in the image would support that rating. +Respond in JSON format: {{"rationales": [{{"label": "", "reasoning": ""}}]}}""" + elif ai_assistant == "keyword": + prompt = f"""Look at this image and identify visual features relevant to the rating task: + +Task: {description} +Rating scale: {size} points, from "{min_label}" (1) to "{max_label}" ({size}) + +Identify key visual elements that would influence the rating. +Respond in JSON format: {{"keywords": ["", ""]}}""" + else: + prompt = f"Analyze this image for: {description}" + + try: + return self.ai_endpoint.query_with_image(prompt, image_data, output_format) + except Exception as e: + logger.error(f"Vision query failed for likert {ai_assistant}: {e}") + + # Fall back to standard text-based generation + data = AnnotationInput( + ai_assistant=ai_assistant, + annotation_type=annotation_type, + text=text, + description=description, + min_label=min_label, + max_label=max_label, + size=size + ) + res = self.ai_endpoint.get_ai(data, output_format) + return res + + def generate_multiselect(self, instance_id: int, annotation_id: int, ai_assistant: str) -> str: + annotation_type = _get_scheme_field(annotation_id, "annotation_type") + description = _get_scheme_field(annotation_id, "description") + labels = _get_scheme_field(annotation_id, "labels") + text = _get_instance_text(instance_id) + + ai_prompt = get_ai_prompt() + output_format = self.model_manager.get_model_class_by_name(ai_prompt[annotation_type].get(ai_assistant).get("output_format")) + + # Check if we should use vision endpoint for image-based content + if self.endpoint_supports_vision and _is_image_url(text): + logger.debug(f"Using vision for multiselect {ai_assistant} on image: {text[:50]}...") + image_data = _get_image_data_from_url(text) + if image_data: + # Format labels for the prompt + label_names = [l.get('name', l) if isinstance(l, dict) else l for l in labels] + labels_str = ', '.join(f'"{name}"' for name in label_names) + + # Build vision-specific prompts based on ai_assistant type + if ai_assistant == "hint": + prompt = f"""Look at this image and help with the following annotation task: + +Task: {description} +Available options (select all that apply): {labels_str} + +Please analyze the image and suggest which options apply. +Respond in JSON format: {{"hint": "", "suggestive_choices": ["", ""]}}""" + elif ai_assistant == "rationale": + prompt = f"""Look at this image and explain the reasoning for each option: + +Task: {description} +Available options: {labels_str} + +For each option, explain what visual evidence supports or contradicts it. +Respond in JSON format: {{"rationales": [{{"label": "