diff --git a/.env b/.env index 91635b7826cc89ce0485ac9517f89c58a0251d79..9e0708ea120f171ac539bfb19fc2fa404c56fea2 100644 --- a/.env +++ b/.env @@ -6,8 +6,8 @@ DOTENV_PUBLIC_KEY="02432e6a4895a1e967061d0588df908fba1730139bdf599d3021034d4035a # dotenvx encrypt -ek "SURREAL_*" # .env -GOOGLE_API_KEY=encrypted:BEpFDkAjzT+MXhXpTo0MtlI+PCdMT2xc+qTQQGmHQifUWx4RxNG8ylRvCCB3jFe4fejMhBMBsDXrpt9OiurXDVvoStO4jl6QuVq1TfWHRrqbBYFCVRRlZnr7mx+MCwkhp8sWHWg= -GROQ_API_KEY=encrypted:BGb7PzEU1NUTXUjCslpPqu9mCgwCO7G5cf2B4jp5AciPl2t6lwEhKvFLknRGzLVPu7RHoVN7Y+D78IhgJf3mDnuk9Oiz6b9KkDzXOpRjmQhO22GVnqPM8WtAgqBQx6BY5FGAaik= +GOOGLE_API_KEY=AIzaSyAFlllmL6s0l-SKezU6sZRQ7ZzNQODgQro +GROQ_API_KEY=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 SURREAL_URL=ws://127.0.0.1:8000/rpc SURREAL_NAMESPACE=open_notebook diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ea76b9fb71fc1aef915ab593ca315cacefb3ebe6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Set PYTHONPATH to include /app +ENV PYTHONPATH=/app + +# Set Hugging Face cache directories (writable in HF Spaces) +ENV HF_HOME=/tmp +ENV TRANSFORMERS_CACHE=/tmp +ENV SENTENCE_TRANSFORMERS_HOME=/tmp + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + build-essential && \ + curl -sSf https://install.surrealdb.com | sh && \ + curl -fsS https://dotenvx.sh | sh && \ + rm -rf /var/lib/apt/lists/* + +# Copy requirements.txt for dependency installation +COPY requirements.txt ./ + +# Install Python dependencies from requirements.txt +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# Explicitly ensure surreal-commands is installed (belt-and-suspenders approach) +# RUN pip install --no-cache-dir surreal-commands>=1.2.0 +# requirements.txt + +# Pre-download sentence-transformers model at build time +# This will be cached in the Docker image +RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')" + +# Copy application code +COPY api/ ./open_notebook/ ./commands/ ./migrations/ ./prompts/ run_api.py start.sh ./ + +# Make start script executable +RUN chmod +x start.sh + +# Set environment variables for SurrealDB connection +ENV SURREAL_URL=ws://localhost:8000/rpc +ENV SURREAL_ADDRESS=localhost +ENV SURREAL_PORT=8000 +ENV SURREAL_USER=root +ENV SURREAL_PASS=root +ENV SURREAL_NAMESPACE=open_notebook +ENV SURREAL_DATABASE=main + +# Set API configuration for Hugging Face Spaces +ENV API_HOST=0.0.0.0 +ENV API_PORT=7860 +ENV API_RELOAD=false + +# Expose Hugging Face Spaces port +EXPOSE 7860 + +# Run the start script +CMD ["dotenvx", "./start.sh"] diff --git a/api/CLAUDE.md b/api/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..8e17a5daeb965dfea44b97fb3ea1070df18dc690 --- /dev/null +++ b/api/CLAUDE.md @@ -0,0 +1,260 @@ +# API Module + +FastAPI-based REST backend exposing services for notebooks, sources, notes, chat, podcasts, and AI model management. + +## Purpose + +FastAPI application serving three architectural layers: routes (HTTP endpoints), services (business logic), and models (request/response schemas). Integrates LangGraph workflows (chat, ask, source_chat), SurrealDB persistence, and AI providers via Esperanto. + +## Architecture Overview + +**Three layers**: +1. **Routes** (`routers/*`): HTTP endpoints mapping to services +2. **Services** (`*_service.py`): Business logic orchestrating domain models, database, graphs, AI providers +3. **Models** (`models.py`): Pydantic request/response schemas with validation + +**Startup flow**: +- Load .env environment variables +- Initialize CORS middleware + password auth middleware +- Run database migrations via AsyncMigrationManager on lifespan startup +- Run podcast profile data migration (legacy string to model registry conversion) +- Register all routers + +**Key services**: +- `chat_service.py`: Invokes chat graph with messages, context +- `podcast_service.py`: Orchestrates outline + transcript generation +- `sources_service.py`: Content ingestion, vectorization, metadata +- `notes_service.py`: Note creation, linking to sources/insights +- `transformations_service.py`: Applies transformations to content +- `models_service.py`: Manages AI provider/model configuration +- `episode_profiles_service.py`: Manages podcast speaker/episode profiles + +## Component Catalog + +### Main Application +- **main.py**: FastAPI app initialization, CORS setup, auth middleware, lifespan event, router registration +- **Lifespan handler**: Runs AsyncMigrationManager on startup (database schema migration) +- **Auth middleware**: PasswordAuthMiddleware protects endpoints (password-based access control) + +### Services (Business Logic) +- **chat_service.py**: Invokes chat.py graph; handles message history via SqliteSaver +- **podcast_service.py**: Generates outline (outline.jinja), then transcript (transcript.jinja) for episodes +- **sources_service.py**: Ingests files/URLs (content_core), extracts text, vectorizes, saves to SurrealDB +- **transformations_service.py**: Applies transformations via transformation.py graph +- **models_service.py**: Manages ModelManager config (AI provider overrides) +- **episode_profiles_service.py**: CRUD for EpisodeProfile and SpeakerProfile models +- **insights_service.py**: Generates and retrieves source insights +- **notes_service.py**: Creates notes linked to sources/insights + +### Models (Schemas) +- **models.py**: Pydantic schemas for request/response validation +- Request bodies: ChatRequest, CreateNoteRequest, PodcastGenerationRequest, etc. +- Response bodies: ChatResponse, NoteResponse, PodcastResponse, etc. +- Custom validators for enum fields, file paths, model references + +### Routers +- **routers/chat.py**: POST /chat +- **routers/source_chat.py**: POST /source/{source_id}/chat +- **routers/podcasts.py**: POST /podcasts, GET /podcasts/{id}, POST /podcasts/episodes/{id}/retry, etc. +- **routers/notes.py**: POST /notes, GET /notes/{id} +- **routers/sources.py**: POST /sources, GET /sources/{id}, DELETE /sources/{id} +- **routers/models.py**: GET /models, POST /models/config +- **routers/credentials.py**: CRUD + test + discover + migrate for credential management +- **routers/transformations.py**: POST /transformations +- **routers/insights.py**: GET /sources/{source_id}/insights +- **routers/auth.py**: POST /auth/password (password-based auth) +- **routers/languages.py**: GET /languages (available podcast languages via pycountry+babel) +- **routers/commands.py**: GET /commands/{command_id} (job status tracking) + +## Common Patterns + +- **Service injection via FastAPI**: Routers import services directly; no DI framework +- **Async/await throughout**: All DB queries, graph invocations, AI calls are async +- **SurrealDB transactions**: Services use repo_query, repo_create, repo_upsert from database layer +- **Config override pattern**: Models/config override via models_service passed to graph.ainvoke(config=...) +- **Error handling**: Custom exception hierarchy (`open_notebook.exceptions`) with global FastAPI exception handlers mapping to HTTP status codes (see Error Handling section below). LangGraph nodes use `classify_error()` to convert raw LLM provider errors into typed exceptions with user-friendly messages. +- **Logging**: loguru logger in main.py; services expected to log key operations +- **Response normalization**: All responses follow standard schema (data + metadata structure) + +## Key Dependencies + +- `fastapi`: FastAPI app, routers, HTTPException +- `pydantic`: Validation models with Field, field_validator +- `open_notebook.graphs`: chat, ask, source_chat, source, transformation graphs +- `open_notebook.database`: SurrealDB repository functions (repo_query, repo_create, repo_upsert) +- `open_notebook.domain`: Notebook, Source, Note, SourceInsight models +- `open_notebook.ai.provision`: provision_langchain_model() factory +- `ai_prompter`: Prompter for template rendering +- `content_core`: extract_content() for file/URL processing +- `esperanto`: AI provider client library (LLM, embeddings, TTS) +- `surreal_commands`: Job queue for async operations (podcast generation) +- `loguru`: Structured logging + +## Important Quirks & Gotchas + +- **Migration auto-run**: Database schema migrations run on every API startup (via lifespan); no manual migration steps +- **PasswordAuthMiddleware is basic**: Uses simple password check; production deployments should replace with OAuth/JWT +- **No request rate limiting**: No built-in rate limiting; deployment must add via proxy/middleware +- **Service state is stateless**: Services don't cache results; each request re-queries database/AI models +- **Graph invocation is blocking**: chat/podcast workflows may take minutes; no timeout handling in services +- **Command job fire-and-forget**: podcast_service.py submits jobs but doesn't wait (async job queue pattern) +- **Model override scoping**: Model config override via RunnableConfig is per-request only (not persistent) +- **CORS open by default**: main.py CORS settings allow all origins (restrict before production) +- **No OpenAPI security scheme**: API docs available without auth (disable before production) +- **Services don't validate user permission**: All endpoints trust authentication layer; no per-notebook permission checks + +## Error Handling + +### Global Exception Handlers (`main.py`) + +FastAPI exception handlers map custom exception types from `open_notebook.exceptions` to HTTP status codes. All error responses include CORS headers. + +| Exception Class | HTTP Status | Use Case | +|----------------|-------------|----------| +| `NotFoundError` | 404 | Resource not found | +| `InvalidInputError` | 400 | Bad request data | +| `AuthenticationError` | 401 | Invalid/missing API key | +| `RateLimitError` | 429 | Provider rate limit exceeded | +| `ConfigurationError` | 422 | Wrong model name, missing config | +| `NetworkError` | 502 | Cannot reach AI provider | +| `ExternalServiceError` | 502 | Provider returned error (500/503, context length) | +| `OpenNotebookError` (base) | 500 | Any other application error | + +### Error Classification (`open_notebook.utils.error_classifier`) + +The `classify_error()` function maps raw exceptions from LLM providers/Esperanto/LangChain into the typed exceptions above with user-friendly messages. Used in all LangGraph graph nodes and SSE streaming handlers. + +**Flow**: Raw exception → keyword matching → `(ExceptionClass, user_message)` → raised → caught by global handler → HTTP response with descriptive message. + +### Frontend Integration + +The frontend `getApiErrorMessage()` helper (`lib/utils/error-handler.ts`) tries i18n mapping first, then falls back to displaying the backend's descriptive error message directly. + +--- + +## How to Add New Endpoint + +1. Create router file in `routers/` (e.g., `routers/new_feature.py`) +2. Import router into `main.py` and register: `app.include_router(new_feature.router, tags=["new_feature"])` +3. Create service in `new_feature_service.py` with business logic +4. Define request/response schemas in `models.py` (or create `new_feature_models.py`) +5. Implement router functions calling service methods +6. Test with `uv run uvicorn api.main:app --host 0.0.0.0 --port 5055` + +## Testing Patterns + +- **Interactive docs**: http://localhost:5055/docs (Swagger UI) +- **Direct service tests**: Import service, call methods directly with test data +- **Mock graphs**: Replace graph.ainvoke() with mock for testing service logic +- **Database: Use test database** (separate SurrealDB instance or mock repo_query) + +--- + +## Credential Management (API Configuration UI) + +The Credential Management system enables users to configure AI provider credentials through the UI instead of environment variables. Keys are stored securely in SurrealDB (encrypted via Fernet) with database-first fallback to environment variables. + +### Router: `routers/credentials.py` + +**Endpoints**: + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/credentials` | List all credentials (optional `?provider=` filter) | +| GET | `/credentials/by-provider/{provider}` | List credentials for a provider | +| POST | `/credentials` | Create a new credential | +| GET | `/credentials/{credential_id}` | Get a specific credential | +| PUT | `/credentials/{credential_id}` | Update a credential | +| DELETE | `/credentials/{credential_id}` | Delete a credential | +| POST | `/credentials/{credential_id}/test` | Test connection using credential | +| POST | `/credentials/{credential_id}/discover` | Discover available models | +| POST | `/credentials/{credential_id}/register-models` | Register discovered models | +| POST | `/credentials/migrate-from-provider-config` | Migrate from legacy ProviderConfig | + +**Supported Providers** (13 total): +- Simple API key: `openai`, `anthropic`, `google`, `groq`, `mistral`, `deepseek`, `xai`, `openrouter`, `voyage`, `elevenlabs` +- URL-based: `ollama` +- Multi-field: `azure`, `vertex`, `openai_compatible` + +**Security Features**: +- NEVER returns actual API key values (only metadata) +- URL validation (SSRF protection) on all URL fields via `_validate_url()` +- Allows private IPs and localhost for self-hosted services (Ollama, LM Studio) +- Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` to be set for storing credentials + +### Domain Model: `Credential` (`open_notebook/domain/credential.py`) + +Individual credential records replacing the old `ProviderConfig` singleton. Each credential stores: +- Provider name, display name, modalities +- Encrypted API key (via Fernet) +- Provider-specific config (base_url, endpoint, api_version, etc.) + +### Integration with Key Provider (`open_notebook/ai/key_provider.py`) + +The `key_provider` module provisions DB-stored credentials into environment variables for Esperanto compatibility: + +**Database-first Pattern**: +1. API endpoint saves keys to `Credential` records (encrypted in SurrealDB) +2. Before model provisioning, `provision_provider_keys(provider)` checks DB, then env vars +3. Keys from DB are set as environment variables for Esperanto compatibility +4. Existing env vars remain unchanged if no DB config exists + +**Key Functions**: +- `get_api_key(provider)`: Get API key (DB first, env fallback) +- `provision_provider_keys(provider)`: Set env vars from DB for a provider +- `provision_all_keys()`: Load all provider keys from DB into env vars + +### Authentication + +No changes to authentication. The `credentials` router uses the same `PasswordAuthMiddleware` as all other endpoints. Keys are protected by the same password-based auth. + +**Auth Flow** (unchanged from `api/auth.py`): +- `PasswordAuthMiddleware`: Global middleware checking `Authorization: Bearer {password}` header +- Default password: `open-notebook-change-me` (set `OPEN_NOTEBOOK_PASSWORD` in production) +- Docker secrets support via `OPEN_NOTEBOOK_PASSWORD_FILE` + +### Connection Testing (`open_notebook/ai/connection_tester.py`) + +The `/credentials/{credential_id}/test` endpoint uses minimal API calls to verify credentials: +- Loads Credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()` +- Uses cheapest/smallest models per provider (TEST_MODELS map) +- Returns success status and descriptive message +- Special handlers for ollama, openai_compatible, and azure providers + +### Migration Workflows + +Two migration endpoints help users transition to the credential system: + +**From environment variables** (`POST /credentials/migrate-from-env`): +1. Checks each provider for env var presence +2. Creates Credential records from env var values +3. Returns summary: migrated, skipped, errors + +**From legacy ProviderConfig** (`POST /credentials/migrate-from-provider-config`): +1. Reads old ProviderConfig records from database +2. Converts each to individual Credential records +3. Returns summary: migrated, skipped, errors + +### Example Usage + +```python +# Check status +GET /credentials/status +# Response: {"configured": {"openai": true, "anthropic": false}, "source": {"openai": "database", "anthropic": "none"}, "encryption_configured": true} + +# Create credential +POST /credentials +{"name": "My OpenAI Key", "provider": "openai", "modalities": ["language", "embedding"], "api_key": "sk-proj-..."} + +# Test connection +POST /credentials/{credential_id}/test +# Response: {"provider": "openai", "success": true, "message": "Connection successful"} + +# Discover models +POST /credentials/{credential_id}/discover +# Response: {"provider": "openai", "models": [{"model_id": "gpt-4", "name": "gpt-4", ...}], "credential_id": "..."} + +# Migrate from env +POST /credentials/migrate-from-env +# Response: {"message": "Migration complete. Migrated 3 providers.", "migrated": ["openai", "anthropic", "groq"], "skipped": [], "errors": []} +``` diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api/auth.py b/api/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..b89058bbcf6c86b4b22952e072098a6f209f6751 --- /dev/null +++ b/api/auth.py @@ -0,0 +1,114 @@ +from typing import Optional + +from fastapi import Depends, HTTPException, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from loguru import logger +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.responses import JSONResponse + +from open_notebook.utils.encryption import get_secret_from_env + + +class PasswordAuthMiddleware(BaseHTTPMiddleware): + """ + Middleware to check password authentication for all API requests. + Always active with default password if OPEN_NOTEBOOK_PASSWORD is not set. + Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. + """ + + def __init__(self, app, excluded_paths: Optional[list] = None): + super().__init__(app) + self.password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD") + self.excluded_paths = excluded_paths or [ + "/", + "/health", + "/docs", + "/openapi.json", + "/redoc", + ] + + async def dispatch(self, request: Request, call_next): + # Skip authentication if no password is set + if not self.password: + return await call_next(request) + + # Skip authentication for excluded paths + if request.url.path in self.excluded_paths: + return await call_next(request) + + # Skip authentication for CORS preflight requests (OPTIONS) + if request.method == "OPTIONS": + return await call_next(request) + + # Check authorization header + auth_header = request.headers.get("Authorization") + + if not auth_header: + return JSONResponse( + status_code=401, + content={"detail": "Missing authorization header"}, + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Expected format: "Bearer {password}" + try: + scheme, credentials = auth_header.split(" ", 1) + if scheme.lower() != "bearer": + raise ValueError("Invalid authentication scheme") + except ValueError: + return JSONResponse( + status_code=401, + content={"detail": "Invalid authorization header format"}, + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Check password + if credentials != self.password: + return JSONResponse( + status_code=401, + content={"detail": "Invalid password"}, + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Password is correct, proceed with the request + response = await call_next(request) + return response + + +# Optional: HTTPBearer security scheme for OpenAPI documentation +security = HTTPBearer(auto_error=False) + + +def check_api_password( + credentials: Optional[HTTPAuthorizationCredentials] = Depends(security), +) -> bool: + """ + Utility function to check API password. + Can be used as a dependency in individual routes if needed. + Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. + Returns True without checking credentials if OPEN_NOTEBOOK_PASSWORD is not configured. + Raises 401 if credentials are missing or don't match the configured password. + """ + password = get_secret_from_env("OPEN_NOTEBOOK_PASSWORD") + + # No password configured - skip authentication + if not password: + return True + + # No credentials provided + if not credentials: + raise HTTPException( + status_code=401, + detail="Missing authorization", + headers={"WWW-Authenticate": "Bearer"}, + ) + + # Check password + if credentials.credentials != password: + raise HTTPException( + status_code=401, + detail="Invalid password", + headers={"WWW-Authenticate": "Bearer"}, + ) + + return True diff --git a/api/chat_service.py b/api/chat_service.py new file mode 100644 index 0000000000000000000000000000000000000000..fa446b8bc82bb6211933a94902bc586c6d220552 --- /dev/null +++ b/api/chat_service.py @@ -0,0 +1,168 @@ +""" +Chat service for API operations. +Provides async interface for chat functionality. +""" + +import os +from typing import Any, Dict, List, Optional + +import httpx +from loguru import logger + + +class ChatService: + """Service for chat-related API operations""" + + def __init__(self): + self.base_url = os.getenv("API_BASE_URL", "http://127.0.0.1:5055") + # Add authentication header if password is set + self.headers = {} + password = os.getenv("OPEN_NOTEBOOK_PASSWORD") + if password: + self.headers["Authorization"] = f"Bearer {password}" + + async def get_sessions(self, notebook_id: str) -> List[Dict[str, Any]]: + """Get all chat sessions for a notebook""" + try: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{self.base_url}/api/chat/sessions", + params={"notebook_id": notebook_id}, + headers=self.headers, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error fetching chat sessions: {str(e)}") + raise + + async def create_session( + self, + notebook_id: str, + title: Optional[str] = None, + model_override: Optional[str] = None, + ) -> Dict[str, Any]: + """Create a new chat session""" + try: + data: Dict[str, Any] = {"notebook_id": notebook_id} + if title is not None: + data["title"] = title + if model_override is not None: + data["model_override"] = model_override + + async with httpx.AsyncClient() as client: + response = await client.post( + f"{self.base_url}/api/chat/sessions", + json=data, + headers=self.headers, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error creating chat session: {str(e)}") + raise + + async def get_session(self, session_id: str) -> Dict[str, Any]: + """Get a specific session with messages""" + try: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{self.base_url}/api/chat/sessions/{session_id}", + headers=self.headers, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error fetching session: {str(e)}") + raise + + async def update_session( + self, + session_id: str, + title: Optional[str] = None, + model_override: Optional[str] = None, + ) -> Dict[str, Any]: + """Update session properties""" + try: + data: Dict[str, Any] = {} + if title is not None: + data["title"] = title + if model_override is not None: + data["model_override"] = model_override + + if not data: + raise ValueError( + "At least one field must be provided to update a session" + ) + + async with httpx.AsyncClient() as client: + response = await client.put( + f"{self.base_url}/api/chat/sessions/{session_id}", + json=data, + headers=self.headers, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error updating session: {str(e)}") + raise + + async def delete_session(self, session_id: str) -> Dict[str, Any]: + """Delete a chat session""" + try: + async with httpx.AsyncClient() as client: + response = await client.delete( + f"{self.base_url}/api/chat/sessions/{session_id}", + headers=self.headers, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error deleting session: {str(e)}") + raise + + async def execute_chat( + self, + session_id: str, + message: str, + context: Dict[str, Any], + model_override: Optional[str] = None, + ) -> Dict[str, Any]: + """Execute a chat request""" + try: + data = {"session_id": session_id, "message": message, "context": context} + if model_override is not None: + data["model_override"] = model_override + + # Short connect timeout (10s), long read timeout (10 min) for Ollama/local LLMs + timeout = httpx.Timeout(connect=10.0, read=600.0, write=30.0, pool=10.0) + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.post( + f"{self.base_url}/api/chat/execute", json=data, headers=self.headers + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error executing chat: {str(e)}") + raise + + async def build_context( + self, notebook_id: str, context_config: Dict[str, Any] + ) -> Dict[str, Any]: + """Build context for a notebook""" + try: + data = {"notebook_id": notebook_id, "context_config": context_config} + + async with httpx.AsyncClient() as client: + response = await client.post( + f"{self.base_url}/api/chat/context", json=data, headers=self.headers + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error building context: {str(e)}") + raise + + +# Global instance +chat_service = ChatService() diff --git a/api/client.py b/api/client.py new file mode 100644 index 0000000000000000000000000000000000000000..91978c061bf5a605677dd5614b645f86530da0e7 --- /dev/null +++ b/api/client.py @@ -0,0 +1,529 @@ +""" +API client for Open Notebook API. +This module provides a client interface to interact with the Open Notebook API. +""" + +import os +from typing import Any, Dict, List, Optional, Union + +import httpx +from loguru import logger + + +class APIClient: + """Client for Open Notebook API.""" + + def __init__(self, base_url: Optional[str] = None): + self.base_url = base_url or os.getenv("API_BASE_URL", "http://127.0.0.1:5055") + # Timeout increased to 5 minutes (300s) to accommodate slow LLM operations + # (transformations, insights) on slower hardware (Ollama, LM Studio, remote APIs) + # Configurable via API_CLIENT_TIMEOUT environment variable (in seconds) + timeout_str = os.getenv("API_CLIENT_TIMEOUT", "300.0") + try: + timeout_value = float(timeout_str) + # Validate timeout is within reasonable bounds (30s - 3600s / 1 hour) + if timeout_value < 30: + logger.warning( + f"API_CLIENT_TIMEOUT={timeout_value}s is too low, using minimum of 30s" + ) + timeout_value = 30.0 + elif timeout_value > 3600: + logger.warning( + f"API_CLIENT_TIMEOUT={timeout_value}s is too high, using maximum of 3600s" + ) + timeout_value = 3600.0 + self.timeout = timeout_value + except ValueError: + logger.error( + f"Invalid API_CLIENT_TIMEOUT value '{timeout_str}', using default 300s" + ) + self.timeout = 300.0 + + # Add authentication header if password is set + self.headers = {} + password = os.getenv("OPEN_NOTEBOOK_PASSWORD") + if password: + self.headers["Authorization"] = f"Bearer {password}" + + def _make_request( + self, method: str, endpoint: str, timeout: Optional[float] = None, **kwargs + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Make HTTP request to the API.""" + url = f"{self.base_url}{endpoint}" + request_timeout = timeout if timeout is not None else self.timeout + + # Merge headers + headers = kwargs.get("headers", {}) + headers.update(self.headers) + kwargs["headers"] = headers + + try: + with httpx.Client(timeout=request_timeout) as client: + response = client.request(method, url, **kwargs) + response.raise_for_status() + return response.json() + except httpx.RequestError as e: + logger.error(f"Request error for {method} {url}: {str(e)}") + raise ConnectionError(f"Failed to connect to API: {str(e)}") + except httpx.HTTPStatusError as e: + logger.error( + f"HTTP error {e.response.status_code} for {method} {url}: {e.response.text}" + ) + raise RuntimeError( + f"API request failed: {e.response.status_code} - {e.response.text}" + ) + except Exception as e: + logger.error(f"Unexpected error for {method} {url}: {str(e)}") + raise + + # Notebooks API methods + def get_notebooks( + self, archived: Optional[bool] = None, order_by: str = "updated desc" + ) -> List[Dict[Any, Any]]: + """Get all notebooks.""" + params: Dict[str, Any] = {"order_by": order_by} + if archived is not None: + params["archived"] = str(archived).lower() + + result = self._make_request("GET", "/api/notebooks", params=params) + return result if isinstance(result, list) else [result] + + def create_notebook( + self, name: str, description: str = "" + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new notebook.""" + data = {"name": name, "description": description} + return self._make_request("POST", "/api/notebooks", json=data) + + def get_notebook( + self, notebook_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific notebook.""" + return self._make_request("GET", f"/api/notebooks/{notebook_id}") + + def update_notebook( + self, notebook_id: str, **updates + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update a notebook.""" + return self._make_request("PUT", f"/api/notebooks/{notebook_id}", json=updates) + + def delete_notebook( + self, notebook_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a notebook.""" + return self._make_request("DELETE", f"/api/notebooks/{notebook_id}") + + # Search API methods + def search( + self, + query: str, + search_type: str = "text", + limit: int = 100, + search_sources: bool = True, + search_notes: bool = True, + minimum_score: float = 0.2, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Search the knowledge base.""" + data = { + "query": query, + "type": search_type, + "limit": limit, + "search_sources": search_sources, + "search_notes": search_notes, + "minimum_score": minimum_score, + } + return self._make_request("POST", "/api/search", json=data) + + def ask_simple( + self, + question: str, + strategy_model: str, + answer_model: str, + final_answer_model: str, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Ask the knowledge base a question (simple, non-streaming).""" + data = { + "question": question, + "strategy_model": strategy_model, + "answer_model": answer_model, + "final_answer_model": final_answer_model, + } + # Use configured timeout for long-running ask operations + return self._make_request( + "POST", "/api/search/ask/simple", json=data, timeout=self.timeout + ) + + # Models API methods + def get_models(self, model_type: Optional[str] = None) -> List[Dict[Any, Any]]: + """Get all models with optional type filtering.""" + params = {} + if model_type: + params["type"] = model_type + result = self._make_request("GET", "/api/models", params=params) + return result if isinstance(result, list) else [result] + + def create_model( + self, name: str, provider: str, model_type: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new model.""" + data = { + "name": name, + "provider": provider, + "type": model_type, + } + return self._make_request("POST", "/api/models", json=data) + + def delete_model( + self, model_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a model.""" + return self._make_request("DELETE", f"/api/models/{model_id}") + + def get_default_models(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get default model assignments.""" + return self._make_request("GET", "/api/models/defaults") + + def update_default_models( + self, **defaults + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update default model assignments.""" + return self._make_request("PUT", "/api/models/defaults", json=defaults) + + # Transformations API methods + def get_transformations(self) -> List[Dict[Any, Any]]: + """Get all transformations.""" + result = self._make_request("GET", "/api/transformations") + return result if isinstance(result, list) else [result] + + def create_transformation( + self, + name: str, + title: str, + description: str, + prompt: str, + apply_default: bool = False, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new transformation.""" + data = { + "name": name, + "title": title, + "description": description, + "prompt": prompt, + "apply_default": apply_default, + } + return self._make_request("POST", "/api/transformations", json=data) + + def get_transformation( + self, transformation_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific transformation.""" + return self._make_request("GET", f"/api/transformations/{transformation_id}") + + def update_transformation( + self, transformation_id: str, **updates + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update a transformation.""" + return self._make_request( + "PUT", f"/api/transformations/{transformation_id}", json=updates + ) + + def delete_transformation( + self, transformation_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a transformation.""" + return self._make_request("DELETE", f"/api/transformations/{transformation_id}") + + def execute_transformation( + self, transformation_id: str, input_text: str, model_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Execute a transformation on input text.""" + data = { + "transformation_id": transformation_id, + "input_text": input_text, + "model_id": model_id, + } + # Use configured timeout for transformation operations + return self._make_request( + "POST", "/api/transformations/execute", json=data, timeout=self.timeout + ) + + # Notes API methods + def get_notes(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]: + """Get all notes with optional notebook filtering.""" + params = {} + if notebook_id: + params["notebook_id"] = notebook_id + result = self._make_request("GET", "/api/notes", params=params) + return result if isinstance(result, list) else [result] + + def create_note( + self, + content: str, + title: Optional[str] = None, + note_type: str = "human", + notebook_id: Optional[str] = None, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new note.""" + data = { + "content": content, + "note_type": note_type, + } + if title: + data["title"] = title + if notebook_id: + data["notebook_id"] = notebook_id + return self._make_request("POST", "/api/notes", json=data) + + def get_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific note.""" + return self._make_request("GET", f"/api/notes/{note_id}") + + def update_note( + self, note_id: str, **updates + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update a note.""" + return self._make_request("PUT", f"/api/notes/{note_id}", json=updates) + + def delete_note(self, note_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a note.""" + return self._make_request("DELETE", f"/api/notes/{note_id}") + + # Embedding API methods + def embed_content( + self, item_id: str, item_type: str, async_processing: bool = False + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Embed content for vector search.""" + data = { + "item_id": item_id, + "item_type": item_type, + "async_processing": async_processing, + } + # Use configured timeout for embedding operations + return self._make_request("POST", "/api/embed", json=data, timeout=self.timeout) + + def rebuild_embeddings( + self, + mode: str = "existing", + include_sources: bool = True, + include_notes: bool = True, + include_insights: bool = True, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Rebuild embeddings in bulk. + + Note: This operation can take a long time for large databases. + Consider increasing API_CLIENT_TIMEOUT to 600-900s for bulk rebuilds. + """ + data = { + "mode": mode, + "include_sources": include_sources, + "include_notes": include_notes, + "include_insights": include_insights, + } + # Use double the configured timeout for bulk rebuild operations (or configured value if already high) + rebuild_timeout = max(self.timeout, min(self.timeout * 2, 3600.0)) + return self._make_request( + "POST", "/api/embeddings/rebuild", json=data, timeout=rebuild_timeout + ) + + def get_rebuild_status( + self, command_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get status of a rebuild operation.""" + return self._make_request("GET", f"/api/embeddings/rebuild/{command_id}/status") + + # Settings API methods + def get_settings(self) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get all application settings.""" + return self._make_request("GET", "/api/settings") + + def update_settings( + self, **settings + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update application settings.""" + return self._make_request("PUT", "/api/settings", json=settings) + + # Context API methods + def get_notebook_context( + self, notebook_id: str, context_config: Optional[Dict] = None + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get context for a notebook.""" + data: Dict[str, Any] = {"notebook_id": notebook_id} + if context_config: + data["context_config"] = context_config + result = self._make_request( + "POST", f"/api/notebooks/{notebook_id}/context", json=data + ) + return result if isinstance(result, dict) else {} + + # Sources API methods + def get_sources(self, notebook_id: Optional[str] = None) -> List[Dict[Any, Any]]: + """Get all sources with optional notebook filtering.""" + params = {} + if notebook_id: + params["notebook_id"] = notebook_id + result = self._make_request("GET", "/api/sources", params=params) + return result if isinstance(result, list) else [result] + + def create_source( + self, + notebook_id: Optional[str] = None, + notebooks: Optional[List[str]] = None, + source_type: str = "text", + url: Optional[str] = None, + file_path: Optional[str] = None, + content: Optional[str] = None, + title: Optional[str] = None, + transformations: Optional[List[str]] = None, + embed: bool = False, + delete_source: bool = False, + async_processing: bool = False, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new source.""" + data = { + "type": source_type, + "embed": embed, + "delete_source": delete_source, + "async_processing": async_processing, + } + + # Handle backward compatibility for notebook_id vs notebooks + if notebooks: + data["notebooks"] = notebooks + elif notebook_id: + data["notebook_id"] = notebook_id + else: + raise ValueError("Either notebook_id or notebooks must be provided") + + if url: + data["url"] = url + if file_path: + data["file_path"] = file_path + if content: + data["content"] = content + if title: + data["title"] = title + if transformations: + data["transformations"] = transformations + + # Use configured timeout for source creation (especially PDF processing with OCR) + return self._make_request( + "POST", "/api/sources/json", json=data, timeout=self.timeout + ) + + def get_source(self, source_id: str) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific source.""" + return self._make_request("GET", f"/api/sources/{source_id}") + + def get_source_status( + self, source_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get processing status for a source.""" + return self._make_request("GET", f"/api/sources/{source_id}/status") + + def update_source( + self, source_id: str, **updates + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update a source.""" + return self._make_request("PUT", f"/api/sources/{source_id}", json=updates) + + def delete_source( + self, source_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a source.""" + return self._make_request("DELETE", f"/api/sources/{source_id}") + + # Insights API methods + def get_source_insights(self, source_id: str) -> List[Dict[Any, Any]]: + """Get all insights for a specific source.""" + result = self._make_request("GET", f"/api/sources/{source_id}/insights") + return result if isinstance(result, list) else [result] + + def get_insight( + self, insight_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific insight.""" + return self._make_request("GET", f"/api/insights/{insight_id}") + + def delete_insight( + self, insight_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete a specific insight.""" + return self._make_request("DELETE", f"/api/insights/{insight_id}") + + def save_insight_as_note( + self, insight_id: str, notebook_id: Optional[str] = None + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Convert an insight to a note.""" + data = {} + if notebook_id: + data["notebook_id"] = notebook_id + return self._make_request( + "POST", f"/api/insights/{insight_id}/save-as-note", json=data + ) + + def create_source_insight( + self, source_id: str, transformation_id: str, model_id: Optional[str] = None + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new insight for a source by running a transformation.""" + data = {"transformation_id": transformation_id} + if model_id: + data["model_id"] = model_id + return self._make_request( + "POST", f"/api/sources/{source_id}/insights", json=data + ) + + # Episode Profiles API methods + def get_episode_profiles(self) -> List[Dict[Any, Any]]: + """Get all episode profiles.""" + result = self._make_request("GET", "/api/episode-profiles") + return result if isinstance(result, list) else [result] + + def get_episode_profile( + self, profile_name: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get a specific episode profile by name.""" + return self._make_request("GET", f"/api/episode-profiles/{profile_name}") + + def create_episode_profile( + self, + name: str, + description: str = "", + speaker_config: str = "", + outline_provider: str = "", + outline_model: str = "", + transcript_provider: str = "", + transcript_model: str = "", + default_briefing: str = "", + num_segments: int = 5, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Create a new episode profile.""" + data = { + "name": name, + "description": description, + "speaker_config": speaker_config, + "outline_provider": outline_provider, + "outline_model": outline_model, + "transcript_provider": transcript_provider, + "transcript_model": transcript_model, + "default_briefing": default_briefing, + "num_segments": num_segments, + } + return self._make_request("POST", "/api/episode-profiles", json=data) + + def update_episode_profile( + self, profile_id: str, **updates + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Update an episode profile.""" + return self._make_request( + "PUT", f"/api/episode-profiles/{profile_id}", json=updates + ) + + def delete_episode_profile( + self, profile_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Delete an episode profile.""" + return self._make_request("DELETE", f"/api/episode-profiles/{profile_id}") + + +# Global client instance +api_client = APIClient() diff --git a/api/command_service.py b/api/command_service.py new file mode 100644 index 0000000000000000000000000000000000000000..6e9506ba21adb477af4f53e7dace797720ee2308 --- /dev/null +++ b/api/command_service.py @@ -0,0 +1,92 @@ +from typing import Any, Dict, List, Optional + +from loguru import logger +from surreal_commands import get_command_status, submit_command + + +class CommandService: + """Generic service layer for command operations""" + + @staticmethod + async def submit_command_job( + module_name: str, # Actually app_name for surreal-commands + command_name: str, + command_args: Dict[str, Any], + context: Optional[Dict[str, Any]] = None, + ) -> str: + """Submit a generic command job for background processing""" + try: + # Ensure command modules are imported before submitting + # This is needed because submit_command validates against local registry + try: + import commands.podcast_commands # noqa: F401 + except ImportError as import_err: + logger.error(f"Failed to import command modules: {import_err}") + raise ValueError("Command modules not available") + + # surreal-commands expects: submit_command(app_name, command_name, args) + cmd_id = submit_command( + module_name, # This is actually the app name (e.g., "open_notebook") + command_name, # Command name (e.g., "process_text") + command_args, # Input data + ) + # Convert RecordID to string if needed + if not cmd_id: + raise ValueError("Failed to get cmd_id from submit_command") + cmd_id_str = str(cmd_id) + logger.info( + f"Submitted command job: {cmd_id_str} for {module_name}.{command_name}" + ) + return cmd_id_str + + except Exception as e: + logger.error(f"Failed to submit command job: {e}") + raise + + @staticmethod + async def get_command_status(job_id: str) -> Dict[str, Any]: + """Get status of any command job""" + try: + status = await get_command_status(job_id) + return { + "job_id": job_id, + "status": status.status if status else "unknown", + "result": status.result if status else None, + "error_message": getattr(status, "error_message", None) + if status + else None, + "created": str(status.created) + if status and hasattr(status, "created") and status.created + else None, + "updated": str(status.updated) + if status and hasattr(status, "updated") and status.updated + else None, + "progress": getattr(status, "progress", None) if status else None, + } + except Exception as e: + logger.error(f"Failed to get command status: {e}") + raise + + @staticmethod + async def list_command_jobs( + module_filter: Optional[str] = None, + command_filter: Optional[str] = None, + status_filter: Optional[str] = None, + limit: int = 50, + ) -> List[Dict[str, Any]]: + """List command jobs with optional filtering""" + # This will be implemented with proper SurrealDB queries + # For now, return empty list as this is foundation phase + return [] + + @staticmethod + async def cancel_command_job(job_id: str) -> bool: + """Cancel a running command job""" + try: + # Implementation depends on surreal-commands cancellation support + # For now, just log the attempt + logger.info(f"Attempting to cancel job: {job_id}") + return True + except Exception as e: + logger.error(f"Failed to cancel command job: {e}") + raise diff --git a/api/context_service.py b/api/context_service.py new file mode 100644 index 0000000000000000000000000000000000000000..87f766fb8b3072e0ff07d70ac3897a4ea0cdbc61 --- /dev/null +++ b/api/context_service.py @@ -0,0 +1,29 @@ +""" +Context service layer using API. +""" + +from typing import Any, Dict, List, Optional, Union + +from loguru import logger + +from api.client import api_client + + +class ContextService: + """Service layer for context operations using API.""" + + def __init__(self): + logger.info("Using API for context operations") + + def get_notebook_context( + self, notebook_id: str, context_config: Optional[Dict] = None + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Get context for a notebook.""" + result = api_client.get_notebook_context( + notebook_id=notebook_id, context_config=context_config + ) + return result + + +# Global service instance +context_service = ContextService() diff --git a/api/credentials_service.py b/api/credentials_service.py new file mode 100644 index 0000000000000000000000000000000000000000..8f288eb702bc3acaeaaab7a8b922f73bd9c554a5 --- /dev/null +++ b/api/credentials_service.py @@ -0,0 +1,890 @@ +""" +Credentials Service + +Business logic for managing AI provider credentials. +Extracted from the credentials router to follow the service layer pattern. + +All functions raise ValueError for business errors (router converts to HTTPException). +""" + +import ipaddress +import os +import socket +from typing import Dict, List, Optional +from urllib.parse import urlparse + +import httpx +from loguru import logger +from pydantic import SecretStr + +from api.models import CredentialResponse +from open_notebook.domain.credential import Credential +from open_notebook.utils.encryption import get_secret_from_env + +# ============================================================================= +# Constants +# ============================================================================= + +# Provider environment variable configuration. +# - "required": ALL listed env vars must be set for the provider to be considered configured. +# - "required_any": at least ONE of the listed env vars must be set. +# - "optional": additional env vars used during migration but not required. +PROVIDER_ENV_CONFIG: Dict[str, dict] = { + "openai": {"required": ["OPENAI_API_KEY"]}, + "anthropic": {"required": ["ANTHROPIC_API_KEY"]}, + "google": {"required_any": ["GOOGLE_API_KEY", "GEMINI_API_KEY"]}, + "groq": {"required": ["GROQ_API_KEY"]}, + "mistral": {"required": ["MISTRAL_API_KEY"]}, + "deepseek": {"required": ["DEEPSEEK_API_KEY"]}, + "xai": {"required": ["XAI_API_KEY"]}, + "openrouter": {"required": ["OPENROUTER_API_KEY"]}, + "voyage": {"required": ["VOYAGE_API_KEY"]}, + "elevenlabs": {"required": ["ELEVENLABS_API_KEY"]}, + "ollama": {"required": ["OLLAMA_API_BASE"]}, + "vertex": { + "required": ["VERTEX_PROJECT", "VERTEX_LOCATION"], + "optional": ["GOOGLE_APPLICATION_CREDENTIALS"], + }, + "azure": { + "required": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION"], + "optional": [ + "AZURE_OPENAI_ENDPOINT_LLM", + "AZURE_OPENAI_ENDPOINT_EMBEDDING", + "AZURE_OPENAI_ENDPOINT_STT", + "AZURE_OPENAI_ENDPOINT_TTS", + ], + }, + "openai_compatible": { + "required_any": ["OPENAI_COMPATIBLE_BASE_URL", "OPENAI_COMPATIBLE_API_KEY"], + }, + "dashscope": {"required": ["DASHSCOPE_API_KEY"]}, + "minimax": {"required": ["MINIMAX_API_KEY"]}, +} + +PROVIDER_MODALITIES: Dict[str, List[str]] = { + "openai": ["language", "embedding", "speech_to_text", "text_to_speech"], + "anthropic": ["language"], + "google": ["language", "embedding"], + "groq": ["language", "speech_to_text"], + "mistral": ["language", "embedding"], + "deepseek": ["language"], + "xai": ["language"], + "openrouter": ["language"], + "voyage": ["embedding"], + "elevenlabs": ["text_to_speech"], + "ollama": ["language", "embedding"], + "vertex": ["language", "embedding"], + "azure": ["language", "embedding", "speech_to_text", "text_to_speech"], + "openai_compatible": ["language", "embedding", "speech_to_text", "text_to_speech"], + "dashscope": ["language"], + "minimax": ["language"], +} + + +# ============================================================================= +# URL Validation (SSRF protection) +# ============================================================================= + + +def validate_url(url: str, provider: str) -> None: + """ + Validate URL format for API endpoints. + + This is a self-hosted application, so we allow: + - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services + - Localhost for local services (Ollama, LM Studio, etc.) + + We only block: + - Invalid schemes (must be http or https) + - Malformed URLs + - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints + - Hostnames that resolve to link-local addresses + + Args: + url: The URL to validate + provider: The provider name (for logging/context) + + Raises: + ValueError: If the URL is invalid + """ + if not url or not url.strip(): + return # Empty URLs handled elsewhere + + try: + parsed = urlparse(url.strip()) + + # Validate scheme - only http/https allowed + if parsed.scheme not in ("http", "https"): + raise ValueError( + f"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed." + ) + + # Extract hostname + hostname = parsed.hostname + if not hostname: + raise ValueError("Invalid URL: hostname could not be determined.") + + # Try to parse as IP address to check for dangerous addresses + try: + ip = ipaddress.ip_address(hostname) + + # Block link-local addresses (169.254.x.x) - used for cloud metadata + # These are dangerous as they can expose cloud instance credentials + if ip.is_link_local: + raise ValueError( + "Link-local addresses (169.254.x.x) are not allowed for security reasons. " + "These addresses are used for cloud metadata endpoints." + ) + + # Block IPv4-mapped IPv6 addresses pointing to link-local + # e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check + if hasattr(ip, "ipv4_mapped") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local: + raise ValueError( + "Link-local addresses (169.254.x.x) are not allowed for security reasons. " + "These addresses are used for cloud metadata endpoints." + ) + + except ValueError as ve: + # Re-raise our own ValueErrors + if "Link-local" in str(ve) or "Invalid URL" in str(ve): + raise + # Not an IP address, it's a hostname - need to resolve and check + try: + # Resolve hostname to IP address + resolved_ips = socket.getaddrinfo(hostname, None) + for family, _, _, _, sockaddr in resolved_ips: + ip_addr = sockaddr[0] + try: + parsed_ip = ipaddress.ip_address(ip_addr) + if parsed_ip.is_link_local: + raise ValueError( + f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. " + "These addresses are used for cloud metadata endpoints." + ) + # Block IPv4-mapped IPv6 addresses pointing to link-local + if ( + hasattr(parsed_ip, "ipv4_mapped") + and parsed_ip.ipv4_mapped + and parsed_ip.ipv4_mapped.is_link_local + ): + raise ValueError( + f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) which is not allowed for security reasons. " + "These addresses are used for cloud metadata endpoints." + ) + except ValueError as inner_ve: + if "link-local" in str(inner_ve).lower() or "Link-local" in str(inner_ve): + raise + # Skip non-IP addresses (e.g., IPv6 zones) + continue + except socket.gaierror: + # Could not resolve hostname - allow it since the URL may be + # valid in the deployment environment (e.g., Azure endpoints, + # internal DNS names). We only block link-local addresses. + pass + + except ValueError: + raise + except Exception: + raise ValueError("Invalid URL format. Check server logs for details.") + + +# ============================================================================= +# Helpers +# ============================================================================= + + +def require_encryption_key() -> None: + """Raise ValueError if encryption key is not configured.""" + if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"): + raise ValueError( + "Encryption key not configured. " + "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to enable storing API keys." + ) + + +def credential_to_response(cred: Credential, model_count: int = 0) -> CredentialResponse: + """Convert a Credential domain object to API response.""" + return CredentialResponse( + id=cred.id or "", + name=cred.name, + provider=cred.provider, + modalities=cred.modalities, + base_url=cred.base_url, + endpoint=cred.endpoint, + api_version=cred.api_version, + endpoint_llm=cred.endpoint_llm, + endpoint_embedding=cred.endpoint_embedding, + endpoint_stt=cred.endpoint_stt, + endpoint_tts=cred.endpoint_tts, + project=cred.project, + location=cred.location, + credentials_path=cred.credentials_path, + has_api_key=cred.api_key is not None, + created=str(cred.created) if cred.created else "", + updated=str(cred.updated) if cred.updated else "", + model_count=model_count, + decryption_error=cred.decryption_error, + ) + + +def check_env_configured(provider: str) -> bool: + """Check if a provider has sufficient env vars configured for migration.""" + config = PROVIDER_ENV_CONFIG.get(provider) + if not config: + return False + + if "required_any" in config: + return any(bool(os.environ.get(v, "").strip()) for v in config["required_any"]) + elif "required" in config: + return all(bool(os.environ.get(v, "").strip()) for v in config["required"]) + return False + + +def get_default_modalities(provider: str) -> List[str]: + """Get default modalities for a provider.""" + return PROVIDER_MODALITIES.get(provider.lower(), ["language"]) + + +def create_credential_from_env(provider: str) -> Credential: + """Create a Credential from environment variables for a given provider.""" + modalities = get_default_modalities(provider) + name = "Default (Migrated from env)" + + if provider == "ollama": + return Credential( + name=name, + provider=provider, + modalities=modalities, + base_url=os.environ.get("OLLAMA_API_BASE"), + ) + elif provider == "vertex": + return Credential( + name=name, + provider=provider, + modalities=modalities, + project=os.environ.get("VERTEX_PROJECT"), + location=os.environ.get("VERTEX_LOCATION"), + credentials_path=os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"), + ) + elif provider == "azure": + return Credential( + name=name, + provider=provider, + modalities=modalities, + api_key=SecretStr(os.environ["AZURE_OPENAI_API_KEY"]), + endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION"), + endpoint_llm=os.environ.get("AZURE_OPENAI_ENDPOINT_LLM"), + endpoint_embedding=os.environ.get("AZURE_OPENAI_ENDPOINT_EMBEDDING"), + endpoint_stt=os.environ.get("AZURE_OPENAI_ENDPOINT_STT"), + endpoint_tts=os.environ.get("AZURE_OPENAI_ENDPOINT_TTS"), + ) + elif provider == "openai_compatible": + api_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") + return Credential( + name=name, + provider=provider, + modalities=modalities, + api_key=SecretStr(api_key) if api_key else None, + base_url=os.environ.get("OPENAI_COMPATIBLE_BASE_URL"), + ) + elif provider == "google": + # Support both GOOGLE_API_KEY and GEMINI_API_KEY (fallback) + api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") + return Credential( + name=name, + provider=provider, + modalities=modalities, + api_key=SecretStr(api_key) if api_key else None, + ) + else: + # Simple API key providers + config = PROVIDER_ENV_CONFIG.get(provider, {}) + required = config.get("required", []) + env_var = required[0] if required else None + api_key = os.environ.get(env_var) if env_var else None + return Credential( + name=name, + provider=provider, + modalities=modalities, + api_key=SecretStr(api_key) if api_key else None, + ) + + +# ============================================================================= +# Service Functions +# ============================================================================= + + +async def get_provider_status() -> dict: + """ + Get configuration status: encryption key status, and per-provider + configured/source information. + """ + encryption_configured = bool(get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY")) + + configured: Dict[str, bool] = {} + source: Dict[str, str] = {} + + for provider in PROVIDER_ENV_CONFIG: + env_configured = check_env_configured(provider) + try: + db_credentials = await Credential.get_by_provider(provider) + db_configured = len(db_credentials) > 0 + except Exception: + db_configured = False + + configured[provider] = db_configured or env_configured + + if db_configured: + source[provider] = "database" + elif env_configured: + source[provider] = "environment" + else: + source[provider] = "none" + + return { + "configured": configured, + "source": source, + "encryption_configured": encryption_configured, + } + + +async def get_env_status() -> Dict[str, bool]: + """Check what's configured via environment variables.""" + env_status: Dict[str, bool] = {} + for provider in PROVIDER_ENV_CONFIG: + env_status[provider] = check_env_configured(provider) + return env_status + + +async def test_credential(credential_id: str) -> dict: + """ + Test connection using a credential's configuration. + + Returns dict with provider, success, message keys. + """ + provider = "unknown" + try: + cred = await Credential.get(credential_id) + config = cred.to_esperanto_config() + + from open_notebook.ai.connection_tester import ( + _test_azure_connection, + _test_ollama_connection, + _test_openai_compatible_connection, + ) + + provider = cred.provider.lower() + + # Handle special providers + if provider == "ollama": + base_url = config.get("base_url", "http://localhost:11434") + success, message = await _test_ollama_connection(base_url) + return {"provider": provider, "success": success, "message": message} + + if provider == "openai_compatible": + base_url = config.get("base_url") + api_key = config.get("api_key") + if not base_url: + return { + "provider": provider, + "success": False, + "message": "No base URL configured", + } + success, message = await _test_openai_compatible_connection( + base_url, api_key + ) + return {"provider": provider, "success": success, "message": message} + + if provider == "azure": + success, message = await _test_azure_connection( + endpoint=config.get("endpoint"), + api_key=config.get("api_key"), + api_version=config.get("api_version"), + ) + return {"provider": provider, "success": success, "message": message} + + # Standard provider: use Esperanto to create and test + from esperanto.factory import AIFactory + + from open_notebook.ai.connection_tester import TEST_MODELS + + if provider not in TEST_MODELS: + return { + "provider": provider, + "success": False, + "message": f"Unknown provider: {provider}", + } + + test_model, test_type = TEST_MODELS[provider] + if not test_model: + return { + "provider": provider, + "success": False, + "message": f"No test model configured for {provider}", + } + + if test_type == "language": + model = AIFactory.create_language( + model_name=test_model, provider=provider, config=config + ) + lc_model = model.to_langchain() + await lc_model.ainvoke("Hi") + return {"provider": provider, "success": True, "message": "Connection successful"} + + elif test_type == "embedding": + model = AIFactory.create_embedding( + model_name=test_model, provider=provider, config=config + ) + await model.aembed(["test"]) + return {"provider": provider, "success": True, "message": "Connection successful"} + + elif test_type == "text_to_speech": + AIFactory.create_text_to_speech(model_name=test_model, provider=provider, config=config) + return { + "provider": provider, + "success": True, + "message": "Connection successful (key format valid)", + } + + return { + "provider": provider, + "success": False, + "message": f"Unsupported test type: {test_type}", + } + + except Exception as e: + error_msg = str(e) + if "401" in error_msg or "unauthorized" in error_msg.lower(): + return {"provider": provider, "success": False, "message": "Invalid API key"} + elif "403" in error_msg or "forbidden" in error_msg.lower(): + return {"provider": provider, "success": False, "message": "API key lacks required permissions"} + elif "rate" in error_msg.lower() and "limit" in error_msg.lower(): + return {"provider": provider, "success": True, "message": "Rate limited - but connection works"} + elif "not found" in error_msg.lower() and "model" in error_msg.lower(): + return {"provider": provider, "success": True, "message": "API key valid (test model not available)"} + else: + logger.debug(f"Test connection error for credential {credential_id}: {e}") + truncated = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg + return {"provider": provider, "success": False, "message": f"Error: {truncated}"} + + +async def discover_with_config(provider: str, config: dict) -> List[dict]: + """ + Discover models using explicit config instead of env vars. + + Returns model names only — no type classification. + The user chooses the model type when registering. + """ + api_key = config.get("api_key") + base_url = config.get("base_url") + + # Static model lists for providers without a listing API + STATIC_MODELS: Dict[str, List[str]] = { + "anthropic": [ + "claude-opus-4-20250514", + "claude-sonnet-4-20250514", + "claude-3-5-sonnet-20241022", + "claude-3-5-haiku-20241022", + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", + ], + "voyage": [ + "voyage-3", "voyage-3-lite", "voyage-code-3", + "voyage-finance-2", "voyage-law-2", "voyage-multilingual-2", + ], + "elevenlabs": [ + "eleven_multilingual_v2", "eleven_turbo_v2_5", + "eleven_turbo_v2", "eleven_monolingual_v1", + ], + } + + if provider in STATIC_MODELS: + if not api_key and provider != "ollama": + return [] + return [ + {"name": m, "provider": provider} + for m in STATIC_MODELS[provider] + ] + + # API-based discovery URLs (OpenAI-style /models endpoints) + url_map = { + "openai": "https://api.openai.com/v1/models", + "groq": "https://api.groq.com/openai/v1/models", + "mistral": "https://api.mistral.ai/v1/models", + "deepseek": "https://api.deepseek.com/models", + "xai": "https://api.x.ai/v1/models", + "openrouter": "https://openrouter.ai/api/v1/models", + "dashscope": "https://dashscope.aliyuncs.com/compatible-mode/v1/models", + "minimax": "https://api.minimax.io/v1/models", + } + + if provider == "ollama": + ollama_url = base_url or "http://localhost:11434" + try: + async with httpx.AsyncClient() as client: + response = await client.get(f"{ollama_url}/api/tags", timeout=10.0) + response.raise_for_status() + data = response.json() + return [ + {"name": m.get("name", ""), "provider": "ollama"} + for m in data.get("models", []) + if m.get("name") + ] + except Exception as e: + logger.warning(f"Failed to discover Ollama models: {e}") + return [] + + if provider == "openai_compatible": + if not base_url: + return [] + try: + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + async with httpx.AsyncClient() as client: + response = await client.get( + f"{base_url.rstrip('/')}/models", headers=headers, timeout=30.0, + ) + response.raise_for_status() + data = response.json() + return [ + {"name": m.get("id", ""), "provider": "openai_compatible"} + for m in data.get("data", []) + if m.get("id") + ] + except Exception as e: + logger.warning(f"Failed to discover openai_compatible models: {e}") + return [] + + if provider == "azure": + endpoint = config.get("endpoint") + api_version = config.get("api_version", "2024-10-21") + if not endpoint or not api_key: + return [] + try: + url = f"{endpoint.rstrip('/')}/openai/models?api-version={api_version}" + headers = {"api-key": api_key} + async with httpx.AsyncClient() as client: + response = await client.get(url, headers=headers, timeout=30.0) + response.raise_for_status() + data = response.json() + return [ + {"name": m.get("id", ""), "provider": "azure"} + for m in data.get("data", []) + if m.get("id") + ] + except Exception as e: + logger.warning(f"Failed to discover Azure models: {e}") + return [] + + if provider == "vertex": + # Vertex AI requires service-account OAuth2 for model listing. + # Return a curated static list of well-known Vertex models instead. + VERTEX_MODELS = [ + "gemini-2.0-flash", + "gemini-2.0-flash-lite", + "gemini-1.5-pro", + "gemini-1.5-flash", + "text-embedding-005", + ] + return [{"name": m, "provider": "vertex"} for m in VERTEX_MODELS] + + if provider == "google": + try: + headers = {"X-Goog-Api-Key": api_key} if api_key else {} + async with httpx.AsyncClient() as client: + response = await client.get( + "https://generativelanguage.googleapis.com/v1/models", + headers=headers, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + return [ + { + "name": model.get("name", "").replace("models/", ""), + "provider": "google", + "description": model.get("displayName"), + } + for model in data.get("models", []) + if model.get("name") + ] + except Exception as e: + logger.warning(f"Failed to discover Google models: {e}") + return [] + + # Standard OpenAI-style API discovery + discovery_url = url_map.get(provider) + if not discovery_url or not api_key: + return [] + + try: + async with httpx.AsyncClient() as client: + response = await client.get( + discovery_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + return [ + { + "name": m.get("id", ""), + "provider": provider, + "description": m.get("name"), + } + for m in data.get("data", []) + if m.get("id") + ] + except Exception as e: + logger.warning(f"Failed to discover {provider} models: {e}") + return [] + + +async def register_models(credential_id: str, models_data: list) -> dict: + """ + Register discovered models and link them to a credential. + + Args: + credential_id: The credential ID to link models to + models_data: List of dicts with name, provider, model_type + + Returns: + dict with created and existing counts + """ + cred = await Credential.get(credential_id) + + from open_notebook.ai.models import Model + from open_notebook.database.repository import repo_query + + # Batch fetch existing models for this provider + existing_models = await repo_query( + "SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model " + "WHERE string::lowercase(provider) = $provider", + {"provider": cred.provider.lower()}, + ) + existing_keys = {(m["name"], m["type"]) for m in existing_models} + + created = 0 + existing = 0 + + for model_data in models_data: + key = (model_data.name.lower(), model_data.model_type.lower()) + if key in existing_keys: + existing += 1 + continue + + new_model = Model( + name=model_data.name, + provider=model_data.provider or cred.provider, + type=model_data.model_type, + credential=cred.id, + ) + await new_model.save() + created += 1 + + return {"created": created, "existing": existing} + + +async def migrate_from_provider_config() -> dict: + """ + Migrate existing ProviderConfig data to individual credential records. + + Returns dict with message, migrated, skipped, errors. + """ + logger.info("=== Starting ProviderConfig migration ===") + + require_encryption_key() + logger.info("Encryption key verified") + + from open_notebook.domain.provider_config import ProviderConfig + + config = await ProviderConfig.get_instance() + logger.info( + f"Found ProviderConfig with {len(config.credentials)} provider(s): " + f"{', '.join(config.credentials.keys())}" + ) + + migrated = [] + skipped = [] + errors = [] + + for provider, credentials_list in config.credentials.items(): + for old_cred in credentials_list: + try: + # Check if a credential already exists for this provider with same name + existing = await Credential.get_by_provider(provider) + names = [c.name for c in existing] + if old_cred.name in names: + logger.info( + f"[{provider}/{old_cred.name}] Already exists in DB, skipping" + ) + skipped.append(f"{provider}/{old_cred.name}") + continue + + # Determine modalities from the provider type + modalities = get_default_modalities(provider) + + logger.info(f"[{provider}/{old_cred.name}] Creating credential") + new_cred = Credential( + name=old_cred.name, + provider=provider, + modalities=modalities, + api_key=old_cred.api_key, + base_url=old_cred.base_url, + endpoint=old_cred.endpoint, + api_version=old_cred.api_version, + endpoint_llm=old_cred.endpoint_llm, + endpoint_embedding=old_cred.endpoint_embedding, + endpoint_stt=old_cred.endpoint_stt, + endpoint_tts=old_cred.endpoint_tts, + project=old_cred.project, + location=old_cred.location, + credentials_path=old_cred.credentials_path, + ) + await new_cred.save() + logger.info( + f"[{provider}/{old_cred.name}] Credential saved (id={new_cred.id})" + ) + + # Link existing models for this provider to the new credential + from open_notebook.ai.models import Model + from open_notebook.database.repository import repo_query + + provider_models = await repo_query( + "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE", + {"provider": provider.lower()}, + ) + if provider_models: + logger.info( + f"[{provider}/{old_cred.name}] Linking {len(provider_models)} " + f"unassigned model(s)" + ) + for model_data in provider_models: + model = Model(**model_data) + model.credential = new_cred.id + await model.save() + + migrated.append(f"{provider}/{old_cred.name}") + + except Exception as e: + logger.error( + f"[{provider}/{old_cred.name}] Migration FAILED: " + f"{type(e).__name__}: {e}", + exc_info=True, + ) + errors.append(f"{provider}/{old_cred.name}: {e}") + + logger.info( + f"=== ProviderConfig migration complete === " + f"migrated={len(migrated)} skipped={len(skipped)} errors={len(errors)}" + ) + if migrated: + logger.info(f" Migrated: {', '.join(migrated)}") + if skipped: + logger.info(f" Skipped: {', '.join(skipped)}") + if errors: + logger.error(f" Errors: {'; '.join(errors)}") + + return { + "message": f"Migration complete. Migrated {len(migrated)} credentials.", + "migrated": migrated, + "skipped": skipped, + "errors": errors, + } + + +async def migrate_from_env() -> dict: + """ + Migrate API keys from environment variables to credential records. + + Returns dict with message, migrated, skipped, not_configured, errors. + """ + logger.info("=== Starting environment variable migration ===") + logger.info( + f"Checking {len(PROVIDER_ENV_CONFIG)} providers: " + f"{', '.join(PROVIDER_ENV_CONFIG.keys())}" + ) + + require_encryption_key() + logger.info("Encryption key verified") + + from open_notebook.ai.models import Model + from open_notebook.database.repository import repo_query + + migrated = [] + skipped = [] + not_configured = [] + errors = [] + + for provider in PROVIDER_ENV_CONFIG: + try: + if not check_env_configured(provider): + logger.debug(f"[{provider}] No env vars configured, skipping") + not_configured.append(provider) + continue + + logger.info(f"[{provider}] Env vars detected, checking for existing credentials") + + existing = await Credential.get_by_provider(provider) + if existing: + logger.info( + f"[{provider}] Already has {len(existing)} credential(s) in DB, skipping" + ) + skipped.append(provider) + continue + + logger.info(f"[{provider}] Creating credential from env vars") + cred = create_credential_from_env(provider) + await cred.save() + logger.info(f"[{provider}] Credential saved successfully (id={cred.id})") + + # Link unassigned models to this credential + provider_models = await repo_query( + "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND credential IS NONE", + {"provider": provider.lower()}, + ) + if provider_models: + logger.info( + f"[{provider}] Linking {len(provider_models)} unassigned model(s) " + f"to credential {cred.id}" + ) + for model_data in provider_models: + model = Model(**model_data) + model.credential = cred.id + await model.save() + else: + logger.info(f"[{provider}] No unassigned models to link") + + migrated.append(provider) + + except Exception as e: + logger.error( + f"[{provider}] Migration FAILED: {type(e).__name__}: {e}", + exc_info=True, + ) + errors.append(f"{provider}: {e}") + + logger.info( + f"=== Environment variable migration complete === " + f"migrated={len(migrated)} skipped={len(skipped)} " + f"not_configured={len(not_configured)} errors={len(errors)}" + ) + if migrated: + logger.info(f" Migrated: {', '.join(migrated)}") + if skipped: + logger.info(f" Skipped (already in DB): {', '.join(skipped)}") + if errors: + logger.error(f" Errors: {'; '.join(errors)}") + + return { + "message": f"Migration complete. Migrated {len(migrated)} providers.", + "migrated": migrated, + "skipped": skipped, + "not_configured": not_configured, + "errors": errors, + } diff --git a/api/embedding_service.py b/api/embedding_service.py new file mode 100644 index 0000000000000000000000000000000000000000..1378d73b8616393355252054037fe5590648b40f --- /dev/null +++ b/api/embedding_service.py @@ -0,0 +1,27 @@ +""" +Embedding service layer using API. +""" + +from typing import Any, Dict, List, Union + +from loguru import logger + +from api.client import api_client + + +class EmbeddingService: + """Service layer for embedding operations using API.""" + + def __init__(self): + logger.info("Using API for embedding operations") + + def embed_content( + self, item_id: str, item_type: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Embed content for vector search.""" + result = api_client.embed_content(item_id=item_id, item_type=item_type) + return result + + +# Global service instance +embedding_service = EmbeddingService() diff --git a/api/episode_profiles_service.py b/api/episode_profiles_service.py new file mode 100644 index 0000000000000000000000000000000000000000..f2072e424369c789e5b37681dac58c7d95cdc5e2 --- /dev/null +++ b/api/episode_profiles_service.py @@ -0,0 +1,112 @@ +""" +Episode profiles service layer using API. +""" + +from typing import List + +from loguru import logger + +from api.client import api_client +from open_notebook.podcasts.models import EpisodeProfile + + +class EpisodeProfilesService: + """Service layer for episode profiles operations using API.""" + + def __init__(self): + logger.info("Using API for episode profiles operations") + + def get_all_episode_profiles(self) -> List[EpisodeProfile]: + """Get all episode profiles.""" + profiles_data = api_client.get_episode_profiles() + # Convert API response to EpisodeProfile objects + profiles = [] + for profile_data in profiles_data: + profile = EpisodeProfile( + name=profile_data["name"], + description=profile_data.get("description", ""), + speaker_config=profile_data["speaker_config"], + outline_provider=profile_data["outline_provider"], + outline_model=profile_data["outline_model"], + transcript_provider=profile_data["transcript_provider"], + transcript_model=profile_data["transcript_model"], + default_briefing=profile_data["default_briefing"], + num_segments=profile_data["num_segments"], + ) + profile.id = profile_data["id"] + profiles.append(profile) + return profiles + + def get_episode_profile(self, profile_name: str) -> EpisodeProfile: + """Get a specific episode profile by name.""" + profile_response = api_client.get_episode_profile(profile_name) + profile_data = ( + profile_response + if isinstance(profile_response, dict) + else profile_response[0] + ) + profile = EpisodeProfile( + name=profile_data["name"], + description=profile_data.get("description", ""), + speaker_config=profile_data["speaker_config"], + outline_provider=profile_data["outline_provider"], + outline_model=profile_data["outline_model"], + transcript_provider=profile_data["transcript_provider"], + transcript_model=profile_data["transcript_model"], + default_briefing=profile_data["default_briefing"], + num_segments=profile_data["num_segments"], + ) + profile.id = profile_data["id"] + return profile + + def create_episode_profile( + self, + name: str, + description: str = "", + speaker_config: str = "", + outline_provider: str = "", + outline_model: str = "", + transcript_provider: str = "", + transcript_model: str = "", + default_briefing: str = "", + num_segments: int = 5, + ) -> EpisodeProfile: + """Create a new episode profile.""" + profile_response = api_client.create_episode_profile( + name=name, + description=description, + speaker_config=speaker_config, + outline_provider=outline_provider, + outline_model=outline_model, + transcript_provider=transcript_provider, + transcript_model=transcript_model, + default_briefing=default_briefing, + num_segments=num_segments, + ) + profile_data = ( + profile_response + if isinstance(profile_response, dict) + else profile_response[0] + ) + profile = EpisodeProfile( + name=profile_data["name"], + description=profile_data.get("description", ""), + speaker_config=profile_data["speaker_config"], + outline_provider=profile_data["outline_provider"], + outline_model=profile_data["outline_model"], + transcript_provider=profile_data["transcript_provider"], + transcript_model=profile_data["transcript_model"], + default_briefing=profile_data["default_briefing"], + num_segments=profile_data["num_segments"], + ) + profile.id = profile_data["id"] + return profile + + def delete_episode_profile(self, profile_id: str) -> bool: + """Delete an episode profile.""" + api_client.delete_episode_profile(profile_id) + return True + + +# Global service instance +episode_profiles_service = EpisodeProfilesService() diff --git a/api/insights_service.py b/api/insights_service.py new file mode 100644 index 0000000000000000000000000000000000000000..6ae876224f778eb60aef7d5986c3429a5003cd98 --- /dev/null +++ b/api/insights_service.py @@ -0,0 +1,100 @@ +""" +Insights service layer using API. +""" + +from typing import List, Optional + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.notebook import Note, SourceInsight + + +class InsightsService: + """Service layer for insights operations using API.""" + + def __init__(self): + logger.info("Using API for insights operations") + + def get_source_insights(self, source_id: str) -> List[SourceInsight]: + """Get all insights for a specific source.""" + insights_data = api_client.get_source_insights(source_id) + # Convert API response to SourceInsight objects + insights = [] + for insight_data in insights_data: + insight = SourceInsight( + insight_type=insight_data["insight_type"], + content=insight_data["content"], + ) + insight.id = insight_data["id"] + insight.created = insight_data["created"] + insight.updated = insight_data["updated"] + insights.append(insight) + return insights + + def get_insight(self, insight_id: str) -> SourceInsight: + """Get a specific insight.""" + insight_response = api_client.get_insight(insight_id) + insight_data = ( + insight_response + if isinstance(insight_response, dict) + else insight_response[0] + ) + insight = SourceInsight( + insight_type=insight_data["insight_type"], + content=insight_data["content"], + ) + insight.id = insight_data["id"] + insight.created = insight_data["created"] + insight.updated = insight_data["updated"] + # Note: source_id from API response is not stored; use await insight.get_source() if needed + return insight + + def delete_insight(self, insight_id: str) -> bool: + """Delete a specific insight.""" + api_client.delete_insight(insight_id) + return True + + def save_insight_as_note( + self, insight_id: str, notebook_id: Optional[str] = None + ) -> Note: + """Convert an insight to a note.""" + note_response = api_client.save_insight_as_note(insight_id, notebook_id) + note_data = ( + note_response if isinstance(note_response, dict) else note_response[0] + ) + note = Note( + title=note_data["title"], + content=note_data["content"], + note_type=note_data["note_type"], + ) + note.id = note_data["id"] + note.created = note_data["created"] + note.updated = note_data["updated"] + return note + + def create_source_insight( + self, source_id: str, transformation_id: str, model_id: Optional[str] = None + ) -> SourceInsight: + """Create a new insight for a source by running a transformation.""" + insight_response = api_client.create_source_insight( + source_id, transformation_id, model_id + ) + insight_data = ( + insight_response + if isinstance(insight_response, dict) + else insight_response[0] + ) + insight = SourceInsight( + insight_type=insight_data["insight_type"], + content=insight_data["content"], + ) + insight.id = insight_data["id"] + insight.created = insight_data["created"] + insight.updated = insight_data["updated"] + # Note: source_id from API response is not stored; use await insight.get_source() if needed + return insight + + +# Global service instance +insights_service = InsightsService() diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000000000000000000000000000000000000..3fbac5ffd5cbd7c7b59f7a3cee3661f7e1b95c0e --- /dev/null +++ b/api/main.py @@ -0,0 +1,322 @@ +# Load environment variables +from dotenv import load_dotenv + +load_dotenv() + +import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from loguru import logger +from starlette.exceptions import HTTPException as StarletteHTTPException + +from api.auth import PasswordAuthMiddleware +from api.routers import ( + auth, + chat, + config, + context, + credentials, + embedding, + embedding_rebuild, + episode_profiles, + insights, + languages, + models, + notebooks, + notes, + podcasts, + search, + settings, + source_chat, + sources, + speaker_profiles, + transformations, +) +from api.routers import commands as commands_router +from open_notebook.database.async_migrate import AsyncMigrationManager +from open_notebook.exceptions import ( + AuthenticationError, + ConfigurationError, + ExternalServiceError, + InvalidInputError, + NetworkError, + NotFoundError, + OpenNotebookError, + RateLimitError, +) +from open_notebook.utils.encryption import get_secret_from_env + + +def _parse_cors_origins(raw: str) -> list[str]: + """Parse CORS_ORIGINS env value into a list of origins.""" + value = raw.strip() + if value == "*": + return ["*"] + return [origin.strip() for origin in value.split(",") if origin.strip()] + + +# Parsed once at module load; CORS_ORIGINS changes require a restart. +_cors_origins_raw = os.getenv("CORS_ORIGINS") +CORS_ALLOWED_ORIGINS = _parse_cors_origins(_cors_origins_raw or "*") +CORS_IS_DEFAULT_WILDCARD = _cors_origins_raw is None + + +def _cors_headers(request: Request) -> dict[str, str]: + """ + Build CORS headers for error responses. + + Mirrors Starlette CORSMiddleware behavior: reflects the request Origin + when the origin is allowed (or when wildcard is configured, since + browsers reject `Access-Control-Allow-Origin: *` combined with + credentials). Omits `Access-Control-Allow-Origin` for disallowed + origins so the browser blocks the error body from leaking cross-origin. + """ + origin = request.headers.get("origin") + headers: dict[str, str] = { + "Access-Control-Allow-Credentials": "true", + "Access-Control-Allow-Methods": "*", + "Access-Control-Allow-Headers": "*", + } + + if origin and ("*" in CORS_ALLOWED_ORIGINS or origin in CORS_ALLOWED_ORIGINS): + headers["Access-Control-Allow-Origin"] = origin + headers["Vary"] = "Origin" + + return headers + + +# Import commands to register them in the API process +try: + logger.info("Commands imported in API process") +except Exception as e: + logger.error(f"Failed to import commands in API process: {e}") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """ + Lifespan event handler for the FastAPI application. + Runs database migrations automatically on startup. + """ + # Startup: Security checks + logger.info("Starting API initialization...") + + # Security check: Encryption key + if not get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY"): + logger.warning( + "OPEN_NOTEBOOK_ENCRYPTION_KEY not set. " + "API key encryption will fail until this is configured. " + "Set OPEN_NOTEBOOK_ENCRYPTION_KEY to any secret string." + ) + + # Run database migrations + + try: + migration_manager = AsyncMigrationManager() + current_version = await migration_manager.get_current_version() + logger.info(f"Current database version: {current_version}") + + if await migration_manager.needs_migration(): + logger.warning("Database migrations are pending. Running migrations...") + await migration_manager.run_migration_up() + new_version = await migration_manager.get_current_version() + logger.success( + f"Migrations completed successfully. Database is now at version {new_version}" + ) + else: + logger.info( + "Database is already at the latest version. No migrations needed." + ) + except Exception as e: + logger.error(f"CRITICAL: Database migration failed: {str(e)}") + logger.exception(e) + # Fail fast - don't start the API with an outdated database schema + raise RuntimeError(f"Failed to run database migrations: {str(e)}") from e + + # Run podcast profile data migration (legacy strings -> Model registry) + try: + from open_notebook.podcasts.migration import migrate_podcast_profiles + + await migrate_podcast_profiles() + except Exception as e: + logger.warning(f"Podcast profile migration encountered errors: {e}") + # Non-fatal: profiles can be migrated manually via UI + + logger.success("API initialization completed successfully") + + # Yield control to the application + yield + + # Shutdown: cleanup if needed + logger.info("API shutdown complete") + + +app = FastAPI( + title="Open Notebook API", + description="API for Open Notebook - Research Assistant", + lifespan=lifespan, +) + +if CORS_IS_DEFAULT_WILDCARD: + logger.warning( + "CORS_ORIGINS is not set — API accepts cross-origin requests from any " + "origin (default: '*'). For production deployments, set CORS_ORIGINS to " + "your frontend origin(s), e.g. " + "CORS_ORIGINS=https://notebook.example.com" + ) +else: + logger.info(f"CORS allowed origins: {CORS_ALLOWED_ORIGINS}") + +# Add password authentication middleware first +# Exclude /api/auth/status and /api/config from authentication +app.add_middleware( + PasswordAuthMiddleware, + excluded_paths=[ + "/", + "/health", + "/docs", + "/openapi.json", + "/redoc", + "/api/auth/status", + "/api/config", + ], +) + +# Add CORS middleware last (so it processes first) +app.add_middleware( + CORSMiddleware, + allow_origins=CORS_ALLOWED_ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +# Custom exception handler to ensure CORS headers are included in error responses +# This helps when errors occur before the CORS middleware can process them +@app.exception_handler(StarletteHTTPException) +async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException): + """ + Custom exception handler that ensures CORS headers are included in error responses. + This is particularly important for 413 (Payload Too Large) errors during file uploads. + + Note: If a reverse proxy (nginx, traefik) returns 413 before the request reaches + FastAPI, this handler won't be called. In that case, configure your reverse proxy + to add CORS headers to error responses. + """ + return JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + headers={**(exc.headers or {}), **_cors_headers(request)}, + ) + + +@app.exception_handler(NotFoundError) +async def not_found_error_handler(request: Request, exc: NotFoundError): + return JSONResponse( + status_code=404, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(InvalidInputError) +async def invalid_input_error_handler(request: Request, exc: InvalidInputError): + return JSONResponse( + status_code=400, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(AuthenticationError) +async def authentication_error_handler(request: Request, exc: AuthenticationError): + return JSONResponse( + status_code=401, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(RateLimitError) +async def rate_limit_error_handler(request: Request, exc: RateLimitError): + return JSONResponse( + status_code=429, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(ConfigurationError) +async def configuration_error_handler(request: Request, exc: ConfigurationError): + return JSONResponse( + status_code=422, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(NetworkError) +async def network_error_handler(request: Request, exc: NetworkError): + return JSONResponse( + status_code=502, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(ExternalServiceError) +async def external_service_error_handler(request: Request, exc: ExternalServiceError): + return JSONResponse( + status_code=502, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +@app.exception_handler(OpenNotebookError) +async def open_notebook_error_handler(request: Request, exc: OpenNotebookError): + return JSONResponse( + status_code=500, + content={"detail": str(exc)}, + headers=_cors_headers(request), + ) + + +# Include routers +app.include_router(auth.router, prefix="/api", tags=["auth"]) +app.include_router(config.router, prefix="/api", tags=["config"]) +app.include_router(notebooks.router, prefix="/api", tags=["notebooks"]) +app.include_router(search.router, prefix="/api", tags=["search"]) +app.include_router(models.router, prefix="/api", tags=["models"]) +app.include_router(transformations.router, prefix="/api", tags=["transformations"]) +app.include_router(notes.router, prefix="/api", tags=["notes"]) +app.include_router(embedding.router, prefix="/api", tags=["embedding"]) +app.include_router( + embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"] +) +app.include_router(settings.router, prefix="/api", tags=["settings"]) +app.include_router(context.router, prefix="/api", tags=["context"]) +app.include_router(sources.router, prefix="/api", tags=["sources"]) +app.include_router(insights.router, prefix="/api", tags=["insights"]) +app.include_router(commands_router.router, prefix="/api", tags=["commands"]) +app.include_router(podcasts.router, prefix="/api", tags=["podcasts"]) +app.include_router(episode_profiles.router, prefix="/api", tags=["episode-profiles"]) +app.include_router(speaker_profiles.router, prefix="/api", tags=["speaker-profiles"]) +app.include_router(chat.router, prefix="/api", tags=["chat"]) +app.include_router(source_chat.router, prefix="/api", tags=["source-chat"]) +app.include_router(credentials.router, prefix="/api", tags=["credentials"]) +app.include_router(languages.router, prefix="/api", tags=["languages"]) + + +@app.get("/") +async def root(): + return {"message": "Open Notebook API is running"} + + +@app.get("/health") +async def health(): + return {"status": "healthy"} diff --git a/api/models.py b/api/models.py new file mode 100644 index 0000000000000000000000000000000000000000..4be004965e1adb9fd9a12e38d71f403bdb0e0709 --- /dev/null +++ b/api/models.py @@ -0,0 +1,686 @@ +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +# Notebook models +class NotebookCreate(BaseModel): + name: str = Field(..., description="Name of the notebook") + description: str = Field(default="", description="Description of the notebook") + + +class NotebookUpdate(BaseModel): + name: Optional[str] = Field(None, description="Name of the notebook") + description: Optional[str] = Field(None, description="Description of the notebook") + archived: Optional[bool] = Field( + None, description="Whether the notebook is archived" + ) + + +class NotebookResponse(BaseModel): + id: str + name: str + description: str + archived: bool + created: str + updated: str + source_count: int + note_count: int + + +# Search models +class SearchRequest(BaseModel): + query: str = Field(..., description="Search query") + type: Literal["text", "vector"] = Field("text", description="Search type") + limit: int = Field(100, description="Maximum number of results", le=1000) + search_sources: bool = Field(True, description="Include sources in search") + search_notes: bool = Field(True, description="Include notes in search") + minimum_score: float = Field( + 0.2, description="Minimum score for vector search", ge=0, le=1 + ) + + +class SearchResponse(BaseModel): + results: List[Dict[str, Any]] = Field(..., description="Search results") + total_count: int = Field(..., description="Total number of results") + search_type: str = Field(..., description="Type of search performed") + + +class AskRequest(BaseModel): + question: str = Field(..., description="Question to ask the knowledge base") + strategy_model: str = Field(..., description="Model ID for query strategy") + answer_model: str = Field(..., description="Model ID for individual answers") + final_answer_model: str = Field(..., description="Model ID for final answer") + + +class AskResponse(BaseModel): + answer: str = Field(..., description="Final answer from the knowledge base") + question: str = Field(..., description="Original question") + + +# Models API models +class ModelCreate(BaseModel): + name: str = Field(..., description="Model name (e.g., gpt-5-mini, claude, gemini)") + provider: str = Field( + ..., description="Provider name (e.g., openai, anthropic, gemini)" + ) + type: str = Field( + ..., + description="Model type (language, embedding, text_to_speech, speech_to_text)", + ) + credential: Optional[str] = Field( + None, description="Credential ID to link this model to" + ) + + +class ModelResponse(BaseModel): + id: str + name: str + provider: str + type: str + credential: Optional[str] = None + created: str + updated: str + + +class DefaultModelsResponse(BaseModel): + default_chat_model: Optional[str] = None + default_transformation_model: Optional[str] = None + large_context_model: Optional[str] = None + default_text_to_speech_model: Optional[str] = None + default_speech_to_text_model: Optional[str] = None + default_embedding_model: Optional[str] = None + default_tools_model: Optional[str] = None + + +class ProviderAvailabilityResponse(BaseModel): + available: List[str] = Field(..., description="List of available providers") + unavailable: List[str] = Field(..., description="List of unavailable providers") + supported_types: Dict[str, List[str]] = Field( + ..., description="Provider to supported model types mapping" + ) + + +# Transformations API models +class TransformationCreate(BaseModel): + name: str = Field(..., description="Transformation name") + title: str = Field(..., description="Display title for the transformation") + description: str = Field( + ..., description="Description of what this transformation does" + ) + prompt: str = Field(..., description="The transformation prompt") + apply_default: bool = Field( + False, description="Whether to apply this transformation by default" + ) + + +class TransformationUpdate(BaseModel): + name: Optional[str] = Field(None, description="Transformation name") + title: Optional[str] = Field( + None, description="Display title for the transformation" + ) + description: Optional[str] = Field( + None, description="Description of what this transformation does" + ) + prompt: Optional[str] = Field(None, description="The transformation prompt") + apply_default: Optional[bool] = Field( + None, description="Whether to apply this transformation by default" + ) + + +class TransformationResponse(BaseModel): + id: str + name: str + title: str + description: str + prompt: str + apply_default: bool + created: str + updated: str + + +class TransformationExecuteRequest(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + + transformation_id: str = Field( + ..., description="ID of the transformation to execute" + ) + input_text: str = Field(..., description="Text to transform") + model_id: str = Field(..., description="Model ID to use for the transformation") + + +class TransformationExecuteResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + + output: str = Field(..., description="Transformed text") + transformation_id: str = Field(..., description="ID of the transformation used") + model_id: str = Field(..., description="Model ID used") + + +# Default Prompt API models +class DefaultPromptResponse(BaseModel): + transformation_instructions: str = Field( + ..., description="Default transformation instructions" + ) + + +class DefaultPromptUpdate(BaseModel): + transformation_instructions: str = Field( + ..., description="Default transformation instructions" + ) + + +# Notes API models +class NoteCreate(BaseModel): + title: Optional[str] = Field(None, description="Note title") + content: str = Field(..., description="Note content") + note_type: Optional[str] = Field("human", description="Type of note (human, ai)") + notebook_id: Optional[str] = Field( + None, description="Notebook ID to add the note to" + ) + + +class NoteUpdate(BaseModel): + title: Optional[str] = Field(None, description="Note title") + content: Optional[str] = Field(None, description="Note content") + note_type: Optional[str] = Field(None, description="Type of note (human, ai)") + + +class NoteResponse(BaseModel): + id: str + title: Optional[str] + content: Optional[str] + note_type: Optional[str] + created: str + updated: str + command_id: Optional[str] = None + + +# Embedding API models +class EmbedRequest(BaseModel): + item_id: str = Field(..., description="ID of the item to embed") + item_type: str = Field(..., description="Type of item (source, note)") + async_processing: bool = Field( + False, description="Process asynchronously in background" + ) + + +class EmbedResponse(BaseModel): + success: bool = Field(..., description="Whether embedding was successful") + message: str = Field(..., description="Result message") + item_id: str = Field(..., description="ID of the item that was embedded") + item_type: str = Field(..., description="Type of item that was embedded") + command_id: Optional[str] = Field( + None, description="Command ID for async processing" + ) + + +# Rebuild request/response models +class RebuildRequest(BaseModel): + mode: Literal["existing", "all"] = Field( + ..., + description="Rebuild mode: 'existing' only re-embeds items with embeddings, 'all' embeds everything", + ) + include_sources: bool = Field(True, description="Include sources in rebuild") + include_notes: bool = Field(True, description="Include notes in rebuild") + include_insights: bool = Field(True, description="Include insights in rebuild") + + +class RebuildResponse(BaseModel): + command_id: str = Field(..., description="Command ID to track progress") + total_items: int = Field(..., description="Estimated number of items to process") + message: str = Field(..., description="Status message") + + +class RebuildProgress(BaseModel): + processed: int = Field(..., description="Number of items processed") + total: int = Field(..., description="Total items to process") + percentage: float = Field(..., description="Progress percentage") + + +class RebuildStats(BaseModel): + sources: int = Field(0, description="Sources processed") + notes: int = Field(0, description="Notes processed") + insights: int = Field(0, description="Insights processed") + failed: int = Field(0, description="Failed items") + + +class RebuildStatusResponse(BaseModel): + command_id: str = Field(..., description="Command ID") + status: str = Field(..., description="Status: queued, running, completed, failed") + progress: Optional[RebuildProgress] = None + stats: Optional[RebuildStats] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + error_message: Optional[str] = None + + +# Settings API models +class SettingsResponse(BaseModel): + default_content_processing_engine_doc: Optional[str] = None + default_content_processing_engine_url: Optional[str] = None + default_embedding_option: Optional[str] = None + auto_delete_files: Optional[str] = None + youtube_preferred_languages: Optional[List[str]] = None + + +class SettingsUpdate(BaseModel): + default_content_processing_engine_doc: Optional[str] = None + default_content_processing_engine_url: Optional[str] = None + default_embedding_option: Optional[str] = None + auto_delete_files: Optional[str] = None + youtube_preferred_languages: Optional[List[str]] = None + + +# Sources API models +class AssetModel(BaseModel): + file_path: Optional[str] = None + url: Optional[str] = None + + +class SourceCreate(BaseModel): + # Backward compatibility: support old single notebook_id + notebook_id: Optional[str] = Field( + None, description="Notebook ID to add the source to (deprecated, use notebooks)" + ) + # New multi-notebook support + notebooks: Optional[List[str]] = Field( + None, description="List of notebook IDs to add the source to" + ) + # Required fields + type: str = Field(..., description="Source type: link, upload, or text") + url: Optional[str] = Field(None, description="URL for link type") + file_path: Optional[str] = Field(None, description="File path for upload type") + content: Optional[str] = Field(None, description="Text content for text type") + title: Optional[str] = Field(None, description="Source title") + transformations: Optional[List[str]] = Field( + default_factory=list, description="Transformation IDs to apply" + ) + embed: bool = Field(False, description="Whether to embed content for vector search") + delete_source: bool = Field( + False, description="Whether to delete uploaded file after processing" + ) + # New async processing support + async_processing: bool = Field( + False, description="Whether to process source asynchronously" + ) + + @model_validator(mode="after") + def validate_notebook_fields(self): + # Ensure only one of notebook_id or notebooks is provided + if self.notebook_id is not None and self.notebooks is not None: + raise ValueError( + "Cannot specify both 'notebook_id' and 'notebooks'. Use 'notebooks' for multi-notebook support." + ) + + # Convert single notebook_id to notebooks array for internal processing + if self.notebook_id is not None: + self.notebooks = [self.notebook_id] + # Keep notebook_id for backward compatibility in response + + # Set empty array if no notebooks specified (allow sources without notebooks) + if self.notebooks is None: + self.notebooks = [] + + return self + + +class SourceUpdate(BaseModel): + title: Optional[str] = Field(None, description="Source title") + topics: Optional[List[str]] = Field(None, description="Source topics") + + +class SourceResponse(BaseModel): + id: str + title: Optional[str] + topics: Optional[List[str]] + asset: Optional[AssetModel] + full_text: Optional[str] + embedded: bool + embedded_chunks: int + file_available: Optional[bool] = None + created: str + updated: str + # New fields for async processing + command_id: Optional[str] = None + status: Optional[str] = None + processing_info: Optional[Dict] = None + # Notebook associations + notebooks: Optional[List[str]] = None + + +class SourceListResponse(BaseModel): + id: str + title: Optional[str] + topics: Optional[List[str]] + asset: Optional[AssetModel] + embedded: bool # Boolean flag indicating if source has embeddings + embedded_chunks: int # Number of embedded chunks + insights_count: int + created: str + updated: str + file_available: Optional[bool] = None + # Status fields for async processing + command_id: Optional[str] = None + status: Optional[str] = None + processing_info: Optional[Dict[str, Any]] = None + + +# Context API models +class ContextConfig(BaseModel): + sources: Dict[str, str] = Field( + default_factory=dict, description="Source inclusion config {source_id: level}" + ) + notes: Dict[str, str] = Field( + default_factory=dict, description="Note inclusion config {note_id: level}" + ) + + +class ContextRequest(BaseModel): + notebook_id: str = Field(..., description="Notebook ID to get context for") + context_config: Optional[ContextConfig] = Field( + None, description="Context configuration" + ) + + +class ContextResponse(BaseModel): + notebook_id: str + sources: List[Dict[str, Any]] = Field(..., description="Source context data") + notes: List[Dict[str, Any]] = Field(..., description="Note context data") + total_tokens: Optional[int] = Field(None, description="Estimated token count") + + +# Insights API models +class SourceInsightResponse(BaseModel): + id: str + source_id: str + insight_type: str + content: str + created: str + updated: str + + +class InsightCreationResponse(BaseModel): + """Response for async insight creation.""" + + status: Literal["pending"] = "pending" + message: str = "Insight generation started" + source_id: str + transformation_id: str + command_id: Optional[str] = None + + +class SaveAsNoteRequest(BaseModel): + notebook_id: Optional[str] = Field(None, description="Notebook ID to add note to") + + +class CreateSourceInsightRequest(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + + transformation_id: str = Field(..., description="ID of transformation to apply") + model_id: Optional[str] = Field( + None, description="Model ID (uses default if not provided)" + ) + + +# Source status response +class SourceStatusResponse(BaseModel): + status: Optional[str] = Field(None, description="Processing status") + message: str = Field(..., description="Descriptive message about the status") + processing_info: Optional[Dict[str, Any]] = Field( + None, description="Detailed processing information" + ) + command_id: Optional[str] = Field(None, description="Command ID if available") + + +# Error response +class ErrorResponse(BaseModel): + error: str + message: str + + +# API Key Configuration models +class SetApiKeyRequest(BaseModel): + """Request to set an API key for a provider.""" + + api_key: Optional[str] = Field(None, description="API key for the provider") + base_url: Optional[str] = Field( + None, description="Base URL for URL-based providers (Ollama, OpenAI-compatible)" + ) + endpoint: Optional[str] = Field( + None, description="Endpoint URL for Azure OpenAI" + ) + api_version: Optional[str] = Field( + None, description="API version for Azure OpenAI" + ) + endpoint_llm: Optional[str] = Field( + None, description="Service-specific endpoint for LLM (Azure)" + ) + endpoint_embedding: Optional[str] = Field( + None, description="Service-specific endpoint for embedding (Azure)" + ) + endpoint_stt: Optional[str] = Field( + None, description="Service-specific endpoint for STT (Azure)" + ) + endpoint_tts: Optional[str] = Field( + None, description="Service-specific endpoint for TTS (Azure)" + ) + service_type: Optional[Literal["llm", "embedding", "stt", "tts"]] = Field( + None, + description="Service type for OpenAI-compatible providers (llm, embedding, stt, tts)", + ) + # Vertex AI specific fields + vertex_project: Optional[str] = Field( + None, description="Google Cloud Project ID for Vertex AI" + ) + vertex_location: Optional[str] = Field( + None, description="Google Cloud Region for Vertex AI (e.g., us-central1)" + ) + vertex_credentials_path: Optional[str] = Field( + None, description="Path to Google Cloud service account JSON file" + ) + + @field_validator( + "api_key", + "base_url", + "endpoint", + "api_version", + "endpoint_llm", + "endpoint_embedding", + "endpoint_stt", + "endpoint_tts", + "vertex_project", + "vertex_location", + "vertex_credentials_path", + mode="before", + ) + @classmethod + def validate_not_empty_string(cls, v: Optional[str]) -> Optional[str]: + """Reject empty strings - convert to None or raise error.""" + if v is not None: + stripped = v.strip() + if not stripped: + return None # Treat empty/whitespace-only as None + return stripped + return v + + +class ApiKeyStatusResponse(BaseModel): + """Response showing which providers are configured and their source.""" + + configured: Dict[str, bool] = Field( + ..., description="Map of provider name to whether it is configured" + ) + source: Dict[str, Literal["database", "environment", "none"]] = Field( + ..., + description="Map of provider name to configuration source (database, environment, or none)", + ) + encryption_configured: bool = Field( + ..., + description="Whether OPEN_NOTEBOOK_ENCRYPTION_KEY is set (required to store keys in database)", + ) + + +class TestConnectionResponse(BaseModel): + """Response from testing a provider connection.""" + + provider: str = Field(..., description="Provider name that was tested") + success: bool = Field(..., description="Whether connection test succeeded") + message: str = Field(..., description="Result message with details") + + +class MigrateFromEnvRequest(BaseModel): + """Request to migrate API keys from environment variables to database.""" + + force: bool = Field( + False, description="Force overwrite existing database configurations" + ) + + +class MigrationResult(BaseModel): + """Response from migrating API keys from environment to database.""" + + message: str = Field(..., description="Summary message") + migrated: List[str] = Field( + default_factory=list, description="Providers successfully migrated" + ) + skipped: List[str] = Field( + default_factory=list, description="Providers skipped (already in DB)" + ) + errors: List[str] = Field( + default_factory=list, description="Migration errors by provider" + ) + + +# Notebook delete cascade models +# Credential models +class CreateCredentialRequest(BaseModel): + """Request to create a new credential.""" + + name: str = Field(..., description="Credential name") + provider: str = Field(..., description="Provider name (openai, anthropic, etc.)") + modalities: List[str] = Field( + default_factory=list, + description="Supported modalities (language, embedding, text_to_speech, speech_to_text)", + ) + api_key: Optional[str] = Field(None, description="API key (stored encrypted)") + base_url: Optional[str] = Field(None, description="Base URL") + endpoint: Optional[str] = Field(None, description="Endpoint URL (Azure)") + api_version: Optional[str] = Field(None, description="API version (Azure)") + endpoint_llm: Optional[str] = Field(None, description="LLM endpoint") + endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint") + endpoint_stt: Optional[str] = Field(None, description="STT endpoint") + endpoint_tts: Optional[str] = Field(None, description="TTS endpoint") + project: Optional[str] = Field(None, description="Project ID (Vertex)") + location: Optional[str] = Field(None, description="Location (Vertex)") + credentials_path: Optional[str] = Field( + None, description="Credentials file path (Vertex)" + ) + + +class UpdateCredentialRequest(BaseModel): + """Request to update an existing credential.""" + + name: Optional[str] = Field(None, description="Credential name") + modalities: Optional[List[str]] = Field(None, description="Supported modalities") + api_key: Optional[str] = Field(None, description="API key (stored encrypted)") + base_url: Optional[str] = Field(None, description="Base URL") + endpoint: Optional[str] = Field(None, description="Endpoint URL") + api_version: Optional[str] = Field(None, description="API version") + endpoint_llm: Optional[str] = Field(None, description="LLM endpoint") + endpoint_embedding: Optional[str] = Field(None, description="Embedding endpoint") + endpoint_stt: Optional[str] = Field(None, description="STT endpoint") + endpoint_tts: Optional[str] = Field(None, description="TTS endpoint") + project: Optional[str] = Field(None, description="Project ID") + location: Optional[str] = Field(None, description="Location") + credentials_path: Optional[str] = Field(None, description="Credentials path") + + +class CredentialResponse(BaseModel): + """Response for a credential (never includes api_key).""" + + id: str + name: str + provider: str + modalities: List[str] + base_url: Optional[str] = None + endpoint: Optional[str] = None + api_version: Optional[str] = None + endpoint_llm: Optional[str] = None + endpoint_embedding: Optional[str] = None + endpoint_stt: Optional[str] = None + endpoint_tts: Optional[str] = None + project: Optional[str] = None + location: Optional[str] = None + credentials_path: Optional[str] = None + has_api_key: bool = False + created: str + updated: str + model_count: int = 0 + decryption_error: Optional[str] = None + + +class CredentialDeleteResponse(BaseModel): + """Response for credential deletion.""" + + message: str + deleted_models: int = 0 + + +class DiscoveredModelResponse(BaseModel): + """A model discovered from a provider.""" + + name: str + provider: str + model_type: Optional[str] = None + description: Optional[str] = None + + +class DiscoverModelsResponse(BaseModel): + """Response from model discovery.""" + + credential_id: str + provider: str + discovered: List[DiscoveredModelResponse] + + +class RegisterModelData(BaseModel): + """A model to register with user-specified type.""" + + name: str + provider: str + model_type: str # Required: user specifies the type + + +class RegisterModelsRequest(BaseModel): + """Request to register discovered models.""" + + models: List[RegisterModelData] + + +class RegisterModelsResponse(BaseModel): + """Response from model registration.""" + + created: int + existing: int + + +class NotebookDeletePreview(BaseModel): + notebook_id: str = Field(..., description="ID of the notebook") + notebook_name: str = Field(..., description="Name of the notebook") + note_count: int = Field(..., description="Number of notes that will be deleted") + exclusive_source_count: int = Field( + ..., description="Number of sources only in this notebook" + ) + shared_source_count: int = Field( + ..., description="Number of sources shared with other notebooks" + ) + + +class NotebookDeleteResponse(BaseModel): + message: str = Field(..., description="Success message") + deleted_notes: int = Field(..., description="Number of notes deleted") + deleted_sources: int = Field(..., description="Number of exclusive sources deleted") + unlinked_sources: int = Field( + ..., description="Number of sources unlinked from notebook" + ) diff --git a/api/models_service.py b/api/models_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ab857fa5c53daeb7fb70a901a0ddc3ff444b1d07 --- /dev/null +++ b/api/models_service.py @@ -0,0 +1,112 @@ +""" +Models service layer using API. +""" + +from typing import List, Optional + +from loguru import logger + +from api.client import api_client +from open_notebook.ai.models import DefaultModels, Model + + +class ModelsService: + """Service layer for models operations using API.""" + + def __init__(self): + logger.info("Using API for models operations") + + def get_all_models(self, model_type: Optional[str] = None) -> List[Model]: + """Get all models with optional type filtering.""" + models_data = api_client.get_models(model_type=model_type) + # Convert API response to Model objects + models = [] + for model_data in models_data: + model = Model( + name=model_data["name"], + provider=model_data["provider"], + type=model_data["type"], + ) + model.id = model_data["id"] + model.created = model_data["created"] + model.updated = model_data["updated"] + models.append(model) + return models + + def create_model(self, name: str, provider: str, model_type: str) -> Model: + """Create a new model.""" + response = api_client.create_model(name, provider, model_type) + model_data = response if isinstance(response, dict) else response[0] + model = Model( + name=model_data["name"], + provider=model_data["provider"], + type=model_data["type"], + ) + model.id = model_data["id"] + model.created = model_data["created"] + model.updated = model_data["updated"] + return model + + def delete_model(self, model_id: str) -> bool: + """Delete a model.""" + api_client.delete_model(model_id) + return True + + def get_default_models(self) -> DefaultModels: + """Get default model assignments.""" + response = api_client.get_default_models() + defaults_data = response if isinstance(response, dict) else response[0] + defaults = DefaultModels() + + # Set the values from API response + defaults.default_chat_model = defaults_data.get("default_chat_model") + defaults.default_transformation_model = defaults_data.get( + "default_transformation_model" + ) + defaults.large_context_model = defaults_data.get("large_context_model") + defaults.default_text_to_speech_model = defaults_data.get( + "default_text_to_speech_model" + ) + defaults.default_speech_to_text_model = defaults_data.get( + "default_speech_to_text_model" + ) + defaults.default_embedding_model = defaults_data.get("default_embedding_model") + defaults.default_tools_model = defaults_data.get("default_tools_model") + + return defaults + + def update_default_models(self, defaults: DefaultModels) -> DefaultModels: + """Update default model assignments.""" + updates = { + "default_chat_model": defaults.default_chat_model, + "default_transformation_model": defaults.default_transformation_model, + "large_context_model": defaults.large_context_model, + "default_text_to_speech_model": defaults.default_text_to_speech_model, + "default_speech_to_text_model": defaults.default_speech_to_text_model, + "default_embedding_model": defaults.default_embedding_model, + "default_tools_model": defaults.default_tools_model, + } + + response = api_client.update_default_models(**updates) + defaults_data = response if isinstance(response, dict) else response[0] + + # Update the defaults object with the response + defaults.default_chat_model = defaults_data.get("default_chat_model") + defaults.default_transformation_model = defaults_data.get( + "default_transformation_model" + ) + defaults.large_context_model = defaults_data.get("large_context_model") + defaults.default_text_to_speech_model = defaults_data.get( + "default_text_to_speech_model" + ) + defaults.default_speech_to_text_model = defaults_data.get( + "default_speech_to_text_model" + ) + defaults.default_embedding_model = defaults_data.get("default_embedding_model") + defaults.default_tools_model = defaults_data.get("default_tools_model") + + return defaults + + +# Global service instance +models_service = ModelsService() diff --git a/api/notebook_service.py b/api/notebook_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ab1ff73858a991ad1b9f52f50919ab771d93f0a9 --- /dev/null +++ b/api/notebook_service.py @@ -0,0 +1,87 @@ +""" +Notebook service layer using API. +""" + +from typing import List, Optional + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.notebook import Notebook + + +class NotebookService: + """Service layer for notebook operations using API.""" + + def __init__(self): + logger.info("Using API for notebook operations") + + def get_all_notebooks(self, order_by: str = "updated desc") -> List[Notebook]: + """Get all notebooks.""" + notebooks_data = api_client.get_notebooks(order_by=order_by) + # Convert API response to Notebook objects + notebooks = [] + for nb_data in notebooks_data: + nb = Notebook( + name=nb_data["name"], + description=nb_data["description"], + archived=nb_data["archived"], + ) + nb.id = nb_data["id"] + nb.created = nb_data["created"] + nb.updated = nb_data["updated"] + notebooks.append(nb) + return notebooks + + def get_notebook(self, notebook_id: str) -> Optional[Notebook]: + """Get a specific notebook.""" + response = api_client.get_notebook(notebook_id) + nb_data = response if isinstance(response, dict) else response[0] + nb = Notebook( + name=nb_data["name"], + description=nb_data["description"], + archived=nb_data["archived"], + ) + nb.id = nb_data["id"] + nb.created = nb_data["created"] + nb.updated = nb_data["updated"] + return nb + + def create_notebook(self, name: str, description: str = "") -> Notebook: + """Create a new notebook.""" + response = api_client.create_notebook(name, description) + nb_data = response if isinstance(response, dict) else response[0] + nb = Notebook( + name=nb_data["name"], + description=nb_data["description"], + archived=nb_data["archived"], + ) + nb.id = nb_data["id"] + nb.created = nb_data["created"] + nb.updated = nb_data["updated"] + return nb + + def update_notebook(self, notebook: Notebook) -> Notebook: + """Update a notebook.""" + updates = { + "name": notebook.name, + "description": notebook.description, + "archived": notebook.archived, + } + response = api_client.update_notebook(notebook.id or "", **updates) + nb_data = response if isinstance(response, dict) else response[0] + # Update the notebook object with the response + notebook.name = nb_data["name"] + notebook.description = nb_data["description"] + notebook.archived = nb_data["archived"] + notebook.updated = nb_data["updated"] + return notebook + + def delete_notebook(self, notebook: Notebook) -> bool: + """Delete a notebook.""" + api_client.delete_notebook(notebook.id or "") + return True + + +# Global service instance +notebook_service = NotebookService() diff --git a/api/notes_service.py b/api/notes_service.py new file mode 100644 index 0000000000000000000000000000000000000000..d336b88d16b6f4b57ce5b44452e6c874adb49968 --- /dev/null +++ b/api/notes_service.py @@ -0,0 +1,103 @@ +""" +Notes service layer using API. +""" + +from typing import List, Optional + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.notebook import Note + + +class NotesService: + """Service layer for notes operations using API.""" + + def __init__(self): + logger.info("Using API for notes operations") + + def get_all_notes(self, notebook_id: Optional[str] = None) -> List[Note]: + """Get all notes with optional notebook filtering.""" + notes_data = api_client.get_notes(notebook_id=notebook_id) + # Convert API response to Note objects + notes = [] + for note_data in notes_data: + note = Note( + title=note_data["title"], + content=note_data["content"], + note_type=note_data["note_type"], + ) + note.id = note_data["id"] + note.created = note_data["created"] + note.updated = note_data["updated"] + notes.append(note) + return notes + + def get_note(self, note_id: str) -> Note: + """Get a specific note.""" + note_response = api_client.get_note(note_id) + note_data = ( + note_response if isinstance(note_response, dict) else note_response[0] + ) + note = Note( + title=note_data["title"], + content=note_data["content"], + note_type=note_data["note_type"], + ) + note.id = note_data["id"] + note.created = note_data["created"] + note.updated = note_data["updated"] + return note + + def create_note( + self, + content: str, + title: Optional[str] = None, + note_type: str = "human", + notebook_id: Optional[str] = None, + ) -> Note: + """Create a new note.""" + note_response = api_client.create_note( + content=content, title=title, note_type=note_type, notebook_id=notebook_id + ) + note_data = ( + note_response if isinstance(note_response, dict) else note_response[0] + ) + note = Note( + title=note_data["title"], + content=note_data["content"], + note_type=note_data["note_type"], + ) + note.id = note_data["id"] + note.created = note_data["created"] + note.updated = note_data["updated"] + return note + + def update_note(self, note: Note) -> Note: + """Update a note.""" + updates = { + "title": note.title, + "content": note.content, + "note_type": note.note_type, + } + note_response = api_client.update_note(note.id or "", **updates) + note_data = ( + note_response if isinstance(note_response, dict) else note_response[0] + ) + + # Update the note object with the response + note.title = note_data["title"] + note.content = note_data["content"] + note.note_type = note_data["note_type"] + note.updated = note_data["updated"] + + return note + + def delete_note(self, note_id: str) -> bool: + """Delete a note.""" + api_client.delete_note(note_id) + return True + + +# Global service instance +notes_service = NotesService() diff --git a/api/podcast_api_service.py b/api/podcast_api_service.py new file mode 100644 index 0000000000000000000000000000000000000000..edf4d847e3bf2354086f109a21a04ef86eda3638 --- /dev/null +++ b/api/podcast_api_service.py @@ -0,0 +1,125 @@ +""" +Podcast service layer using API client. +This replaces direct httpx calls in the Streamlit pages. +""" + +from typing import Any, Dict, List + +from loguru import logger + +from api.client import api_client + + +class PodcastAPIService: + """Service layer for podcast operations using API client.""" + + def __init__(self): + logger.info("Using API client for podcast operations") + + # Episode methods + def get_episodes(self) -> List[Dict[Any, Any]]: + """Get all podcast episodes.""" + result = api_client._make_request("GET", "/api/podcasts/episodes") + return result if isinstance(result, list) else [result] + + def delete_episode(self, episode_id: str) -> bool: + """Delete a podcast episode.""" + try: + api_client._make_request("DELETE", f"/api/podcasts/episodes/{episode_id}") + return True + except Exception as e: + logger.error(f"Failed to delete episode: {e}") + return False + + # Episode Profile methods + def get_episode_profiles(self) -> List[Dict]: + """Get all episode profiles.""" + return api_client.get_episode_profiles() + + def create_episode_profile(self, profile_data: Dict) -> bool: + """Create a new episode profile.""" + try: + api_client.create_episode_profile(**profile_data) + return True + except Exception as e: + logger.error(f"Failed to create episode profile: {e}") + return False + + def update_episode_profile(self, profile_id: str, profile_data: Dict) -> bool: + """Update an episode profile.""" + try: + api_client.update_episode_profile(profile_id, **profile_data) + return True + except Exception as e: + logger.error(f"Failed to update episode profile: {e}") + return False + + def delete_episode_profile(self, profile_id: str) -> bool: + """Delete an episode profile.""" + try: + api_client.delete_episode_profile(profile_id) + return True + except Exception as e: + logger.error(f"Failed to delete episode profile: {e}") + return False + + def duplicate_episode_profile(self, profile_id: str) -> bool: + """Duplicate an episode profile.""" + try: + api_client._make_request( + "POST", f"/api/episode-profiles/{profile_id}/duplicate" + ) + return True + except Exception as e: + logger.error(f"Failed to duplicate episode profile: {e}") + return False + + # Speaker Profile methods + def get_speaker_profiles(self) -> List[Dict[Any, Any]]: + """Get all speaker profiles.""" + result = api_client._make_request("GET", "/api/speaker-profiles") + return result if isinstance(result, list) else [result] + + def create_speaker_profile(self, profile_data: Dict) -> bool: + """Create a new speaker profile.""" + try: + api_client._make_request("POST", "/api/speaker-profiles", json=profile_data) + return True + except Exception as e: + logger.error(f"Failed to create speaker profile: {e}") + return False + + def update_speaker_profile(self, profile_id: str, profile_data: Dict) -> bool: + """Update a speaker profile.""" + try: + api_client._make_request( + "PUT", f"/api/speaker-profiles/{profile_id}", json=profile_data + ) + return True + except Exception as e: + logger.error(f"Failed to update speaker profile: {e}") + return False + + def delete_speaker_profile(self, profile_id: str) -> bool: + """Delete a speaker profile.""" + try: + api_client._make_request("DELETE", f"/api/speaker-profiles/{profile_id}") + return True + except Exception as e: + logger.error(f"Failed to delete speaker profile: {e}") + return False + + def duplicate_speaker_profile(self, profile_id: str) -> bool: + """Duplicate a speaker profile.""" + try: + api_client._make_request( + "POST", f"/api/speaker-profiles/{profile_id}/duplicate" + ) + return True + except Exception as e: + logger.error(f"Failed to duplicate speaker profile: {e}") + return False + + +# Global service instance +podcast_api_service = PodcastAPIService() diff --git a/api/podcast_service.py b/api/podcast_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e7c6dc201c21b0bbef52441213c94a88807cca71 --- /dev/null +++ b/api/podcast_service.py @@ -0,0 +1,206 @@ +from typing import Any, Dict, Optional + +from fastapi import HTTPException +from loguru import logger +from pydantic import BaseModel +from surreal_commands import get_command_status, submit_command + +from open_notebook.domain.notebook import Notebook +from open_notebook.podcasts.models import EpisodeProfile, PodcastEpisode, SpeakerProfile + + +class PodcastGenerationRequest(BaseModel): + """Request model for podcast generation""" + + episode_profile: str + speaker_profile: str + episode_name: str + content: Optional[str] = None + notebook_id: Optional[str] = None + briefing_suffix: Optional[str] = None + + +class PodcastGenerationResponse(BaseModel): + """Response model for podcast generation""" + + job_id: str + status: str + message: str + episode_profile: str + episode_name: str + + +class PodcastService: + """Service layer for podcast operations""" + + @staticmethod + async def submit_generation_job( + episode_profile_name: str, + speaker_profile_name: str, + episode_name: str, + notebook_id: Optional[str] = None, + content: Optional[str] = None, + briefing_suffix: Optional[str] = None, + ) -> str: + """Submit a podcast generation job for background processing""" + try: + # Validate episode profile exists + episode_profile = await EpisodeProfile.get_by_name(episode_profile_name) + if not episode_profile: + raise ValueError(f"Episode profile '{episode_profile_name}' not found") + + # Validate speaker profile exists + speaker_profile = await SpeakerProfile.get_by_name(speaker_profile_name) + if not speaker_profile: + raise ValueError(f"Speaker profile '{speaker_profile_name}' not found") + + # Get content from notebook if not provided directly + if not content and notebook_id: + try: + notebook = await Notebook.get(notebook_id) + # Get notebook context (this may need to be adjusted based on actual Notebook implementation) + content = ( + await notebook.get_context() + if hasattr(notebook, "get_context") + else str(notebook) + ) + except Exception as e: + logger.warning( + f"Failed to get notebook content, using notebook_id as content: {e}" + ) + content = f"Notebook ID: {notebook_id}" + + if not content: + raise ValueError( + "Content is required - provide either content or notebook_id" + ) + + # Prepare command arguments + command_args = { + "episode_profile": episode_profile_name, + "speaker_profile": speaker_profile_name, + "episode_name": episode_name, + "content": str(content), + "briefing_suffix": briefing_suffix, + } + + # Ensure command modules are imported before submitting + # This is needed because submit_command validates against local registry + try: + import commands.podcast_commands # noqa: F401 + except ImportError as import_err: + logger.error(f"Failed to import podcast commands: {import_err}") + raise ValueError("Podcast commands not available") + + # Submit command to surreal-commands + job_id = submit_command("open_notebook", "generate_podcast", command_args) + + # Convert RecordID to string if needed + if not job_id: + raise ValueError("Failed to get job_id from submit_command") + job_id_str = str(job_id) + logger.info( + f"Submitted podcast generation job: {job_id_str} for episode '{episode_name}'" + ) + return job_id_str + + except Exception as e: + logger.error(f"Failed to submit podcast generation job: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to submit podcast generation job: {str(e)}", + ) + + @staticmethod + async def get_job_status(job_id: str) -> Dict[str, Any]: + """Get status of a podcast generation job""" + try: + status = await get_command_status(job_id) + return { + "job_id": job_id, + "status": status.status if status else "unknown", + "result": status.result if status else None, + "error_message": getattr(status, "error_message", None) + if status + else None, + "created": str(status.created) + if status and hasattr(status, "created") and status.created + else None, + "updated": str(status.updated) + if status and hasattr(status, "updated") and status.updated + else None, + "progress": getattr(status, "progress", None) if status else None, + } + except Exception as e: + logger.error(f"Failed to get podcast job status: {e}") + raise HTTPException( + status_code=500, detail=f"Failed to get job status: {str(e)}" + ) + + @staticmethod + async def list_episodes() -> list: + """List all podcast episodes""" + try: + episodes = await PodcastEpisode.get_all(order_by="created desc") + return episodes + except Exception as e: + logger.error(f"Failed to list podcast episodes: {e}") + raise HTTPException( + status_code=500, detail=f"Failed to list episodes: {str(e)}" + ) + + @staticmethod + async def get_episode(episode_id: str) -> PodcastEpisode: + """Get a specific podcast episode""" + try: + episode = await PodcastEpisode.get(episode_id) + return episode + except Exception as e: + logger.error(f"Failed to get podcast episode {episode_id}: {e}") + raise HTTPException(status_code=404, detail=f"Episode not found: {str(e)}") + + +class DefaultProfiles: + """Utility class for creating default profiles (if needed beyond migration data)""" + + @staticmethod + async def create_default_episode_profiles(): + """Create default episode profiles if they don't exist""" + try: + # Check if profiles already exist + existing = await EpisodeProfile.get_all() + if existing: + logger.info(f"Episode profiles already exist: {len(existing)} found") + return existing + + # This would create profiles, but since we have migration data, + # this is mainly for future extensibility + logger.info( + "Default episode profiles should be created via database migration" + ) + return [] + + except Exception as e: + logger.error(f"Failed to create default episode profiles: {e}") + raise + + @staticmethod + async def create_default_speaker_profiles(): + """Create default speaker profiles if they don't exist""" + try: + # Check if profiles already exist + existing = await SpeakerProfile.get_all() + if existing: + logger.info(f"Speaker profiles already exist: {len(existing)} found") + return existing + + # This would create profiles, but since we have migration data, + # this is mainly for future extensibility + logger.info( + "Default speaker profiles should be created via database migration" + ) + return [] + + except Exception as e: + logger.error(f"Failed to create default speaker profiles: {e}") + raise diff --git a/api/routers/__init__.py b/api/routers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api/routers/auth.py b/api/routers/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..363323a1522c887322faf835ff0dcc1141a10aa9 --- /dev/null +++ b/api/routers/auth.py @@ -0,0 +1,27 @@ +""" +Authentication router for Open Notebook API. +Provides endpoints to check authentication status. +""" + +from fastapi import APIRouter + +from open_notebook.utils.encryption import get_secret_from_env + +router = APIRouter(prefix="/auth", tags=["auth"]) + + +@router.get("/status") +async def get_auth_status(): + """ + Check if authentication is enabled. + Returns whether a password is required to access the API. + Supports Docker secrets via OPEN_NOTEBOOK_PASSWORD_FILE. + """ + auth_enabled = bool(get_secret_from_env("OPEN_NOTEBOOK_PASSWORD")) + + return { + "auth_enabled": auth_enabled, + "message": "Authentication is required" + if auth_enabled + else "Authentication is disabled", + } \ No newline at end of file diff --git a/api/routers/chat.py b/api/routers/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..abeeab82d7f3c242d118b3760036469b679ab737 --- /dev/null +++ b/api/routers/chat.py @@ -0,0 +1,526 @@ +import asyncio +import traceback +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException, Query +from langchain_core.runnables import RunnableConfig +from loguru import logger +from pydantic import BaseModel, Field + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.notebook import ChatSession, Note, Notebook, Source +from open_notebook.exceptions import ( + NotFoundError, +) +from open_notebook.graphs.chat import graph as chat_graph +from open_notebook.utils.graph_utils import get_session_message_count + +router = APIRouter() + + +# Request/Response models +class CreateSessionRequest(BaseModel): + notebook_id: str = Field(..., description="Notebook ID to create session for") + title: Optional[str] = Field(None, description="Optional session title") + model_override: Optional[str] = Field( + None, description="Optional model override for this session" + ) + + +class UpdateSessionRequest(BaseModel): + title: Optional[str] = Field(None, description="New session title") + model_override: Optional[str] = Field( + None, description="Model override for this session" + ) + + +class ChatMessage(BaseModel): + id: str = Field(..., description="Message ID") + type: str = Field(..., description="Message type (human|ai)") + content: str = Field(..., description="Message content") + timestamp: Optional[str] = Field(None, description="Message timestamp") + + +class ChatSessionResponse(BaseModel): + id: str = Field(..., description="Session ID") + title: str = Field(..., description="Session title") + notebook_id: Optional[str] = Field(None, description="Notebook ID") + created: str = Field(..., description="Creation timestamp") + updated: str = Field(..., description="Last update timestamp") + message_count: Optional[int] = Field( + None, description="Number of messages in session" + ) + model_override: Optional[str] = Field( + None, description="Model override for this session" + ) + + +class ChatSessionWithMessagesResponse(ChatSessionResponse): + messages: List[ChatMessage] = Field( + default_factory=list, description="Session messages" + ) + + +class ExecuteChatRequest(BaseModel): + session_id: str = Field(..., description="Chat session ID") + message: str = Field(..., description="User message content") + context: Dict[str, Any] = Field( + ..., description="Chat context with sources and notes" + ) + model_override: Optional[str] = Field( + None, description="Optional model override for this message" + ) + + +class ExecuteChatResponse(BaseModel): + session_id: str = Field(..., description="Session ID") + messages: List[ChatMessage] = Field(..., description="Updated message list") + + +class BuildContextRequest(BaseModel): + notebook_id: str = Field(..., description="Notebook ID") + context_config: Dict[str, Any] = Field(..., description="Context configuration") + + +class BuildContextResponse(BaseModel): + context: Dict[str, Any] = Field(..., description="Built context data") + token_count: int = Field(..., description="Estimated token count") + char_count: int = Field(..., description="Character count") + + +class SuccessResponse(BaseModel): + success: bool = Field(True, description="Operation success status") + message: str = Field(..., description="Success message") + + +@router.get("/chat/sessions", response_model=List[ChatSessionResponse]) +async def get_sessions(notebook_id: str = Query(..., description="Notebook ID")): + """Get all chat sessions for a notebook.""" + try: + # Get notebook to verify it exists + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Get sessions for this notebook + sessions_list = await notebook.get_chat_sessions() + + results = [] + for session in sessions_list: + session_id = str(session.id) + + # Get message count from LangGraph state + msg_count = await get_session_message_count(chat_graph, session_id) + + results.append( + ChatSessionResponse( + id=session.id or "", + title=session.title or "Untitled Session", + notebook_id=notebook_id, + created=str(session.created), + updated=str(session.updated), + message_count=msg_count, + model_override=getattr(session, "model_override", None), + ) + ) + + return results + except NotFoundError: + raise HTTPException(status_code=404, detail="Notebook not found") + except Exception as e: + logger.error(f"Error fetching chat sessions: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching chat sessions: {str(e)}" + ) + + +@router.post("/chat/sessions", response_model=ChatSessionResponse) +async def create_session(request: CreateSessionRequest): + """Create a new chat session.""" + try: + # Verify notebook exists + notebook = await Notebook.get(request.notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Create new session + session = ChatSession( + title=request.title + or f"Chat Session {asyncio.get_event_loop().time():.0f}", + model_override=request.model_override, + ) + await session.save() + + # Relate session to notebook + await session.relate_to_notebook(request.notebook_id) + + return ChatSessionResponse( + id=session.id or "", + title=session.title or "", + notebook_id=request.notebook_id, + created=str(session.created), + updated=str(session.updated), + message_count=0, + model_override=session.model_override, + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Notebook not found") + except Exception as e: + logger.error(f"Error creating chat session: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error creating chat session: {str(e)}" + ) + + +@router.get( + "/chat/sessions/{session_id}", response_model=ChatSessionWithMessagesResponse +) +async def get_session(session_id: str): + """Get a specific session with its messages.""" + try: + # Get session + # Ensure session_id has proper table prefix + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Get session state from LangGraph to retrieve messages + # Use sync get_state() in a thread since SqliteSaver doesn't support async + thread_state = await asyncio.to_thread( + chat_graph.get_state, + config=RunnableConfig(configurable={"thread_id": full_session_id}), + ) + + # Extract messages from state + messages: list[ChatMessage] = [] + if thread_state and thread_state.values and "messages" in thread_state.values: + for msg in thread_state.values["messages"]: + messages.append( + ChatMessage( + id=getattr(msg, "id", f"msg_{len(messages)}"), + type=msg.type if hasattr(msg, "type") else "unknown", + content=msg.content if hasattr(msg, "content") else str(msg), + timestamp=None, # LangChain messages don't have timestamps by default + ) + ) + + # Find notebook_id (we need to query the relationship) + # Ensure session_id has proper table prefix + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + + notebook_query = await repo_query( + "SELECT out FROM refers_to WHERE in = $session_id", + {"session_id": ensure_record_id(full_session_id)}, + ) + + notebook_id = notebook_query[0]["out"] if notebook_query else None + + if not notebook_id: + # This might be an old session created before API migration + logger.warning( + f"No notebook relationship found for session {session_id} - may be an orphaned session" + ) + + return ChatSessionWithMessagesResponse( + id=session.id or "", + title=session.title or "Untitled Session", + notebook_id=notebook_id, + created=str(session.created), + updated=str(session.updated), + message_count=len(messages), + messages=messages, + model_override=getattr(session, "model_override", None), + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Session not found") + except Exception as e: + logger.error(f"Error fetching session: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}") + + +@router.put("/chat/sessions/{session_id}", response_model=ChatSessionResponse) +async def update_session(session_id: str, request: UpdateSessionRequest): + """Update session title.""" + try: + # Ensure session_id has proper table prefix + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + update_data = request.model_dump(exclude_unset=True) + + if "title" in update_data: + session.title = update_data["title"] + + if "model_override" in update_data: + session.model_override = update_data["model_override"] + + await session.save() + + # Find notebook_id + # Ensure session_id has proper table prefix + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + notebook_query = await repo_query( + "SELECT out FROM refers_to WHERE in = $session_id", + {"session_id": ensure_record_id(full_session_id)}, + ) + notebook_id = notebook_query[0]["out"] if notebook_query else None + + # Get message count from LangGraph state + msg_count = await get_session_message_count(chat_graph, full_session_id) + + return ChatSessionResponse( + id=session.id or "", + title=session.title or "", + notebook_id=notebook_id, + created=str(session.created), + updated=str(session.updated), + message_count=msg_count, + model_override=session.model_override, + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Session not found") + except Exception as e: + logger.error(f"Error updating session: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error updating session: {str(e)}") + + +@router.delete("/chat/sessions/{session_id}", response_model=SuccessResponse) +async def delete_session(session_id: str): + """Delete a chat session.""" + try: + # Ensure session_id has proper table prefix + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + await session.delete() + + return SuccessResponse(success=True, message="Session deleted successfully") + except NotFoundError: + raise HTTPException(status_code=404, detail="Session not found") + except Exception as e: + logger.error(f"Error deleting session: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}") + + +@router.post("/chat/execute", response_model=ExecuteChatResponse) +async def execute_chat(request: ExecuteChatRequest): + """Execute a chat request and get AI response.""" + try: + # Verify session exists + # Ensure session_id has proper table prefix + full_session_id = ( + request.session_id + if request.session_id.startswith("chat_session:") + else f"chat_session:{request.session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Fetch notebook linked to this session + notebook_query = await repo_query( + "SELECT out FROM refers_to WHERE in = $session_id", + {"session_id": ensure_record_id(full_session_id)}, + ) + notebook = None + if notebook_query: + notebook = await Notebook.get(notebook_query[0]["out"]) + + # Determine model override (per-request override takes precedence over session-level) + model_override = ( + request.model_override + if request.model_override is not None + else getattr(session, "model_override", None) + ) + + # Get current state + # Use sync get_state() in a thread since SqliteSaver doesn't support async + current_state = await asyncio.to_thread( + chat_graph.get_state, + config=RunnableConfig(configurable={"thread_id": full_session_id}), + ) + + # Prepare state for execution + state_values = current_state.values if current_state else {} + state_values["messages"] = state_values.get("messages", []) + state_values["context"] = request.context + state_values["notebook"] = notebook + state_values["model_override"] = model_override + + # Add user message to state + from langchain_core.messages import HumanMessage + + user_message = HumanMessage(content=request.message) + state_values["messages"].append(user_message) + + # Execute chat graph + result = chat_graph.invoke( + input=state_values, # type: ignore[arg-type] + config=RunnableConfig( + configurable={ + "thread_id": full_session_id, + "model_id": model_override, + } + ), + ) + + # Update session timestamp + await session.save() + + # Convert messages to response format + messages: list[ChatMessage] = [] + for msg in result.get("messages", []): + messages.append( + ChatMessage( + id=getattr(msg, "id", f"msg_{len(messages)}"), + type=msg.type if hasattr(msg, "type") else "unknown", + content=msg.content if hasattr(msg, "content") else str(msg), + timestamp=None, + ) + ) + + return ExecuteChatResponse(session_id=request.session_id, messages=messages) + except NotFoundError: + raise HTTPException(status_code=404, detail="Session not found") + except Exception as e: + # Log detailed error with context for debugging + logger.error( + f"Error executing chat: {str(e)}\n" + f" Session ID: {request.session_id}\n" + f" Model override: {request.model_override}\n" + f" Traceback:\n{traceback.format_exc()}" + ) + raise HTTPException(status_code=500, detail=f"Error executing chat: {str(e)}") + + +@router.post("/chat/context", response_model=BuildContextResponse) +async def build_context(request: BuildContextRequest): + """Build context for a notebook based on context configuration.""" + try: + # Verify notebook exists + notebook = await Notebook.get(request.notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []} + total_content = "" + + # Process context configuration if provided + if request.context_config: + # Process sources + for source_id, status in request.context_config.get("sources", {}).items(): + if "not in" in status: + continue + + try: + # Add table prefix if not present + full_source_id = ( + source_id + if source_id.startswith("source:") + else f"source:{source_id}" + ) + + try: + source = await Source.get(full_source_id) + except Exception: + continue + + if "insights" in status: + source_context = await source.get_context(context_size="short") + context_data["sources"].append(source_context) + total_content += str(source_context) + elif "full content" in status: + source_context = await source.get_context(context_size="long") + context_data["sources"].append(source_context) + total_content += str(source_context) + except Exception as e: + logger.warning(f"Error processing source {source_id}: {str(e)}") + continue + + # Process notes + for note_id, status in request.context_config.get("notes", {}).items(): + if "not in" in status: + continue + + try: + # Add table prefix if not present + full_note_id = ( + note_id if note_id.startswith("note:") else f"note:{note_id}" + ) + note = await Note.get(full_note_id) + if not note: + continue + + if "full content" in status: + note_context = note.get_context(context_size="long") + context_data["notes"].append(note_context) + total_content += str(note_context) + except Exception as e: + logger.warning(f"Error processing note {note_id}: {str(e)}") + continue + else: + # Default behavior - include all sources and notes with short context + sources = await notebook.get_sources() + for source in sources: + try: + source_context = await source.get_context(context_size="short") + context_data["sources"].append(source_context) + total_content += str(source_context) + except Exception as e: + logger.warning(f"Error processing source {source.id}: {str(e)}") + continue + + notes = await notebook.get_notes() + for note in notes: + try: + note_context = note.get_context(context_size="short") + context_data["notes"].append(note_context) + total_content += str(note_context) + except Exception as e: + logger.warning(f"Error processing note {note.id}: {str(e)}") + continue + + # Calculate character and token counts + char_count = len(total_content) + # Use token count utility if available + try: + from open_notebook.utils import token_count + + estimated_tokens = token_count(total_content) if total_content else 0 + except ImportError: + # Fallback to simple estimation + estimated_tokens = char_count // 4 + + return BuildContextResponse( + context=context_data, token_count=estimated_tokens, char_count=char_count + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error building context: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error building context: {str(e)}") diff --git a/api/routers/commands.py b/api/routers/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..8b2d41b2b354a76504b48646ed4a55c1ff4ceecc --- /dev/null +++ b/api/routers/commands.py @@ -0,0 +1,166 @@ +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException, Query +from loguru import logger +from pydantic import BaseModel, Field +from surreal_commands import registry + +from api.command_service import CommandService + +router = APIRouter() + + +class CommandExecutionRequest(BaseModel): + command: str = Field( + ..., description="Command function name (e.g., 'process_text')" + ) + app: str = Field(..., description="Application name (e.g., 'open_notebook')") + input: Dict[str, Any] = Field(..., description="Arguments to pass to the command") + + +class CommandJobResponse(BaseModel): + job_id: str + status: str + message: str + + +class CommandJobStatusResponse(BaseModel): + job_id: str + status: str + result: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + created: Optional[str] = None + updated: Optional[str] = None + progress: Optional[Dict[str, Any]] = None + + +@router.post("/commands/jobs", response_model=CommandJobResponse) +async def execute_command(request: CommandExecutionRequest): + """ + Submit a command for background processing. + Returns immediately with job ID for status tracking. + + Example request: + { + "command": "process_text", + "app": "open_notebook", + "input": { + "text": "Hello world", + "operation": "uppercase" + } + } + """ + try: + # Submit command using app name (not module name) + job_id = await CommandService.submit_command_job( + module_name=request.app, # This should be "open_notebook" + command_name=request.command, + command_args=request.input, + ) + + return CommandJobResponse( + job_id=job_id, + status="submitted", + message=f"Command '{request.command}' submitted successfully", + ) + + except Exception as e: + logger.error(f"Error submitting command: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to submit command" + ) + + +@router.get("/commands/jobs/{job_id}", response_model=CommandJobStatusResponse) +async def get_command_job_status(job_id: str): + """Get the status of a specific command job""" + try: + status_data = await CommandService.get_command_status(job_id) + return CommandJobStatusResponse(**status_data) + + except Exception as e: + logger.error(f"Error fetching job status: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to fetch job status" + ) + + +@router.get("/commands/jobs", response_model=List[Dict[str, Any]]) +async def list_command_jobs( + command_filter: Optional[str] = Query(None, description="Filter by command name"), + status_filter: Optional[str] = Query(None, description="Filter by status"), + limit: int = Query(50, description="Maximum number of jobs to return"), +): + """List command jobs with optional filtering""" + try: + jobs = await CommandService.list_command_jobs( + command_filter=command_filter, status_filter=status_filter, limit=limit + ) + return jobs + + except Exception as e: + logger.error(f"Error listing command jobs: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to list command jobs" + ) + + +@router.delete("/commands/jobs/{job_id}") +async def cancel_command_job(job_id: str): + """Cancel a running command job""" + try: + success = await CommandService.cancel_command_job(job_id) + return {"job_id": job_id, "cancelled": success} + + except Exception as e: + logger.error(f"Error cancelling command job: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to cancel command job" + ) + + +@router.get("/commands/registry/debug") +async def debug_registry(): + """Debug endpoint to see what commands are registered""" + try: + # Get all registered commands + all_items = registry.get_all_commands() + + # Create JSON-serializable data + command_items = [] + for item in all_items: + try: + command_items.append( + { + "app_id": item.app_id, + "name": item.name, + "full_id": f"{item.app_id}.{item.name}", + } + ) + except Exception as item_error: + logger.error(f"Error processing item: {item_error}") + + # Get the basic command structure + try: + commands_dict: dict[str, list[str]] = {} + for item in all_items: + if item.app_id not in commands_dict: + commands_dict[item.app_id] = [] + commands_dict[item.app_id].append(item.name) + except Exception: + commands_dict = {} + + return { + "total_commands": len(all_items), + "commands_by_app": commands_dict, + "command_items": command_items, + } + + except Exception as e: + logger.error(f"Error debugging registry: {str(e)}") + return { + "error": str(e), + "total_commands": 0, + "commands_by_app": {}, + "command_items": [], + } diff --git a/api/routers/config.py b/api/routers/config.py new file mode 100644 index 0000000000000000000000000000000000000000..0f5424d1370b49f586b2296bcf50ac1021aec075 --- /dev/null +++ b/api/routers/config.py @@ -0,0 +1,160 @@ +import asyncio +import os +import time +import tomllib +from pathlib import Path +from typing import Optional + +from fastapi import APIRouter, Request +from loguru import logger + +from open_notebook.database.repository import repo_query +from open_notebook.utils.version_utils import ( + compare_versions, + get_version_from_github_async, +) + +router = APIRouter() + +# In-memory cache for version check results +_version_cache: dict = { + "latest_version": None, + "has_update": False, + "timestamp": 0, + "check_failed": False, +} + +# Cache TTL in seconds (24 hours) +VERSION_CACHE_TTL = 24 * 60 * 60 + + +def get_version() -> str: + """Read version from pyproject.toml""" + try: + pyproject_path = Path(__file__).parent.parent.parent / "pyproject.toml" + with open(pyproject_path, "rb") as f: + pyproject = tomllib.load(f) + return pyproject.get("project", {}).get("version", "unknown") + except Exception as e: + logger.warning(f"Could not read version from pyproject.toml: {e}") + return "unknown" + + +async def get_latest_version_cached(current_version: str) -> tuple[Optional[str], bool]: + """ + Check for the latest version from GitHub with caching. + + Returns: + tuple: (latest_version, has_update) + - latest_version: str or None if check failed + - has_update: bool indicating if update is available + """ + global _version_cache + + # Check if cache is still valid (within TTL) + cache_age = time.time() - _version_cache["timestamp"] + if _version_cache["timestamp"] > 0 and cache_age < VERSION_CACHE_TTL: + logger.debug(f"Using cached version check result (age: {cache_age:.0f}s)") + return _version_cache["latest_version"], _version_cache["has_update"] + + # Cache expired or not yet set + if _version_cache["timestamp"] > 0: + logger.info(f"Version cache expired (age: {cache_age:.0f}s), refreshing...") + + # Perform version check with strict error handling + try: + logger.info("Checking for latest version from GitHub...") + + # Fetch latest version from GitHub with 10-second timeout + latest_version = await get_version_from_github_async( + "https://github.com/lfnovo/open-notebook", "main" + ) + + logger.info( + f"Latest version from GitHub: {latest_version}, Current version: {current_version}" + ) + + # Compare versions + has_update = compare_versions(current_version, latest_version) < 0 + + # Cache the result + _version_cache["latest_version"] = latest_version + _version_cache["has_update"] = has_update + _version_cache["timestamp"] = time.time() + _version_cache["check_failed"] = False + + logger.info(f"Version check complete. Update available: {has_update}") + + return latest_version, has_update + + except Exception as e: + logger.warning(f"Version check failed: {e}") + + # Cache the failure to avoid repeated attempts + _version_cache["latest_version"] = None + _version_cache["has_update"] = False + _version_cache["timestamp"] = time.time() + _version_cache["check_failed"] = True + + return None, False + + +async def check_database_health() -> dict: + """ + Check if database is reachable using a lightweight query. + + Returns: + dict with 'status' ("online" | "offline") and optional 'error' + """ + try: + # 2-second timeout for database health check + result = await asyncio.wait_for(repo_query("RETURN 1"), timeout=2.0) + if result: + return {"status": "online"} + return {"status": "offline", "error": "Empty result"} + except asyncio.TimeoutError: + logger.warning("Database health check timed out after 2 seconds") + return {"status": "offline", "error": "Health check timeout"} + except Exception as e: + logger.warning(f"Database health check failed: {e}") + return {"status": "offline", "error": str(e)} + + +@router.get("/config") +async def get_config(request: Request): + """ + Get frontend configuration. + + Returns version information and health status. + Note: The frontend determines the API URL via its own runtime-config endpoint, + so this endpoint no longer returns apiUrl. + + Also checks for version updates from GitHub (with caching and error handling). + """ + # Get current version + current_version = get_version() + + # Check for updates (with caching and error handling) + # This MUST NOT break the endpoint - wrapped in try-except as extra safety + latest_version = None + has_update = False + + try: + latest_version, has_update = await get_latest_version_cached(current_version) + except Exception as e: + # Extra safety: ensure version check never breaks the config endpoint + logger.error(f"Unexpected error during version check: {e}") + + # Check database health + db_health = await check_database_health() + db_status = db_health["status"] + + if db_status == "offline": + logger.warning(f"Database offline: {db_health.get('error', 'Unknown error')}") + + return { + "version": current_version, + "latestVersion": latest_version, + "hasUpdate": has_update, + "dbStatus": db_status, + } diff --git a/api/routers/context.py b/api/routers/context.py new file mode 100644 index 0000000000000000000000000000000000000000..92dd723b2bff0b3765890f551748d059c4461df6 --- /dev/null +++ b/api/routers/context.py @@ -0,0 +1,115 @@ +from fastapi import APIRouter, HTTPException +from loguru import logger + +from api.models import ContextRequest, ContextResponse +from open_notebook.domain.notebook import Note, Notebook, Source +from open_notebook.exceptions import InvalidInputError +from open_notebook.utils import token_count + +router = APIRouter() + + +@router.post("/notebooks/{notebook_id}/context", response_model=ContextResponse) +async def get_notebook_context(notebook_id: str, context_request: ContextRequest): + """Get context for a notebook based on configuration.""" + try: + # Verify notebook exists + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + context_data: dict[str, list[dict[str, str]]] = {"note": [], "source": []} + total_content = "" + + # Process context configuration if provided + if context_request.context_config: + # Process sources + for source_id, status in context_request.context_config.sources.items(): + if "not in" in status: + continue + + try: + # Add table prefix if not present + full_source_id = ( + source_id + if source_id.startswith("source:") + else f"source:{source_id}" + ) + + try: + source = await Source.get(full_source_id) + except Exception: + continue + + if "insights" in status: + source_context = await source.get_context(context_size="short") + context_data["source"].append(source_context) + total_content += str(source_context) + elif "full content" in status: + source_context = await source.get_context(context_size="long") + context_data["source"].append(source_context) + total_content += str(source_context) + except Exception as e: + logger.warning(f"Error processing source {source_id}: {str(e)}") + continue + + # Process notes + for note_id, status in context_request.context_config.notes.items(): + if "not in" in status: + continue + + try: + # Add table prefix if not present + full_note_id = ( + note_id if note_id.startswith("note:") else f"note:{note_id}" + ) + note = await Note.get(full_note_id) + if not note: + continue + + if "full content" in status: + note_context = note.get_context(context_size="long") + context_data["note"].append(note_context) + total_content += str(note_context) + except Exception as e: + logger.warning(f"Error processing note {note_id}: {str(e)}") + continue + else: + # Default behavior - include all sources and notes with short context + sources = await notebook.get_sources() + for source in sources: + try: + source_context = await source.get_context(context_size="short") + context_data["source"].append(source_context) + total_content += str(source_context) + except Exception as e: + logger.warning(f"Error processing source {source.id}: {str(e)}") + continue + + notes = await notebook.get_notes() + for note in notes: + try: + note_context = note.get_context(context_size="short") + context_data["note"].append(note_context) + total_content += str(note_context) + except Exception as e: + logger.warning(f"Error processing note {note.id}: {str(e)}") + continue + + # Calculate estimated token count + estimated_tokens = token_count(total_content) if total_content else 0 + + return ContextResponse( + notebook_id=notebook_id, + sources=context_data["source"], + notes=context_data["note"], + total_tokens=estimated_tokens, + ) + + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error getting context for notebook {notebook_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error getting context: {str(e)}") diff --git a/api/routers/credentials.py b/api/routers/credentials.py new file mode 100644 index 0000000000000000000000000000000000000000..57bbb839f11a44637928c028e72c620a7bfd876e --- /dev/null +++ b/api/routers/credentials.py @@ -0,0 +1,426 @@ +""" +Credentials Router + +Thin HTTP layer for managing individual AI provider credentials. +Business logic lives in api.credentials_service. + +Endpoints: +- GET /credentials - List all credentials +- GET /credentials/by-provider/{provider} - List credentials for a provider +- POST /credentials - Create a new credential +- GET /credentials/{credential_id} - Get a specific credential +- PUT /credentials/{credential_id} - Update a credential +- DELETE /credentials/{credential_id} - Delete a credential +- POST /credentials/{credential_id}/test - Test connection +- POST /credentials/{credential_id}/discover - Discover models +- POST /credentials/{credential_id}/register-models - Register models + +NEVER returns actual API key values - only metadata. +""" + +from typing import List, Optional + +from fastapi import APIRouter, HTTPException, Query +from loguru import logger +from pydantic import SecretStr + +from api.credentials_service import ( + credential_to_response, + discover_with_config, + get_provider_status, + register_models, + require_encryption_key, + validate_url, +) +from api.credentials_service import ( + get_env_status as svc_get_env_status, +) +from api.credentials_service import ( + migrate_from_env as svc_migrate_from_env, +) +from api.credentials_service import ( + migrate_from_provider_config as svc_migrate_from_provider_config, +) +from api.credentials_service import ( + test_credential as svc_test_credential, +) +from api.models import ( + CreateCredentialRequest, + CredentialDeleteResponse, + CredentialResponse, + DiscoveredModelResponse, + DiscoverModelsResponse, + RegisterModelsRequest, + RegisterModelsResponse, + UpdateCredentialRequest, +) +from open_notebook.database.repository import ensure_record_id, repo_delete, repo_query +from open_notebook.domain.credential import Credential + +router = APIRouter(prefix="/credentials", tags=["credentials"]) + + +def _handle_value_error(e: ValueError, status_code: int = 400) -> HTTPException: + """Convert a ValueError from the service layer to an HTTPException.""" + return HTTPException(status_code=status_code, detail=str(e)) + + +# ============================================================================= +# Status endpoints +# ============================================================================= + + +@router.get("/status") +async def get_status(): + """ + Get configuration status: encryption key status, and per-provider + configured/source information. + """ + try: + return await get_provider_status() + except Exception as e: + logger.error(f"Error fetching status: {e}") + raise HTTPException(status_code=500, detail="Failed to fetch credential status") + + +@router.get("/env-status") +async def get_env_status(): + """Check what's configured via environment variables.""" + try: + return await svc_get_env_status() + except Exception as e: + logger.error(f"Error checking env status: {e}") + raise HTTPException(status_code=500, detail="Failed to check environment status") + + +# ============================================================================= +# CRUD endpoints +# ============================================================================= + + +@router.get("", response_model=List[CredentialResponse]) +async def list_credentials( + provider: Optional[str] = Query(None, description="Filter by provider"), +): + """List all credentials, optionally filtered by provider.""" + try: + if provider: + credentials = await Credential.get_by_provider(provider) + else: + credentials = await Credential.get_all(order_by="provider, created") + + result = [] + for cred in credentials: + models = await cred.get_linked_models() + result.append(credential_to_response(cred, len(models))) + + return result + + except Exception as e: + logger.error(f"Error listing credentials: {e}") + raise HTTPException(status_code=500, detail="Failed to list credentials") + + +@router.get("/by-provider/{provider}", response_model=List[CredentialResponse]) +async def list_credentials_by_provider(provider: str): + """List all credentials for a specific provider.""" + try: + credentials = await Credential.get_by_provider(provider.lower()) + result = [] + for cred in credentials: + models = await cred.get_linked_models() + result.append(credential_to_response(cred, len(models))) + return result + except Exception as e: + logger.error(f"Error listing credentials for {provider}: {e}") + raise HTTPException(status_code=500, detail="Failed to list credentials for provider") + + +@router.post("", response_model=CredentialResponse, status_code=201) +async def create_credential(request: CreateCredentialRequest): + """Create a new credential.""" + try: + require_encryption_key() + except ValueError as e: + raise _handle_value_error(e) + + # Validate all URL fields + for url_field in [ + request.base_url, request.endpoint, request.endpoint_llm, + request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts, + ]: + if url_field: + try: + validate_url(url_field, request.provider) + except ValueError as e: + raise _handle_value_error(e) + + try: + cred = Credential( + name=request.name, + provider=request.provider.lower(), + modalities=request.modalities, + api_key=SecretStr(request.api_key) if request.api_key else None, + base_url=request.base_url, + endpoint=request.endpoint, + api_version=request.api_version, + endpoint_llm=request.endpoint_llm, + endpoint_embedding=request.endpoint_embedding, + endpoint_stt=request.endpoint_stt, + endpoint_tts=request.endpoint_tts, + project=request.project, + location=request.location, + credentials_path=request.credentials_path, + ) + await cred.save() + return credential_to_response(cred, 0) + + except Exception as e: + logger.error(f"Error creating credential: {e}") + raise HTTPException(status_code=500, detail="Failed to create credential") + + +@router.get("/{credential_id}", response_model=CredentialResponse) +async def get_credential(credential_id: str): + """Get a specific credential by ID. Never returns api_key.""" + try: + cred = await Credential.get(credential_id) + models = await cred.get_linked_models() + return credential_to_response(cred, len(models)) + except Exception as e: + logger.error(f"Error fetching credential {credential_id}: {e}") + raise HTTPException(status_code=404, detail="Credential not found") + + +@router.put("/{credential_id}", response_model=CredentialResponse) +async def update_credential(credential_id: str, request: UpdateCredentialRequest): + """Update an existing credential.""" + try: + require_encryption_key() + except ValueError as e: + raise _handle_value_error(e) + + # Validate all URL fields being updated + for url_field in [ + request.base_url, request.endpoint, request.endpoint_llm, + request.endpoint_embedding, request.endpoint_stt, request.endpoint_tts, + ]: + if url_field: + try: + validate_url(url_field, "update") + except ValueError as e: + raise _handle_value_error(e) + + try: + cred = await Credential.get(credential_id) + + if request.name is not None: + cred.name = request.name + if request.modalities is not None: + cred.modalities = request.modalities + if request.api_key is not None: + cred.api_key = SecretStr(request.api_key) + if request.base_url is not None: + cred.base_url = request.base_url or None + if request.endpoint is not None: + cred.endpoint = request.endpoint or None + if request.api_version is not None: + cred.api_version = request.api_version or None + if request.endpoint_llm is not None: + cred.endpoint_llm = request.endpoint_llm or None + if request.endpoint_embedding is not None: + cred.endpoint_embedding = request.endpoint_embedding or None + if request.endpoint_stt is not None: + cred.endpoint_stt = request.endpoint_stt or None + if request.endpoint_tts is not None: + cred.endpoint_tts = request.endpoint_tts or None + if request.project is not None: + cred.project = request.project or None + if request.location is not None: + cred.location = request.location or None + if request.credentials_path is not None: + cred.credentials_path = request.credentials_path or None + + await cred.save() + models = await cred.get_linked_models() + return credential_to_response(cred, len(models)) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating credential {credential_id}: {e}") + raise HTTPException(status_code=500, detail="Failed to update credential") + + +@router.delete("/{credential_id}", response_model=CredentialDeleteResponse) +async def delete_credential( + credential_id: str, + migrate_to: Optional[str] = Query( + None, description="Migrate linked models to this credential ID" + ), +): + """ + Delete a credential. + + If the credential has linked models: + - Pass migrate_to= to reassign them to another credential + - Otherwise, linked models are cascade-deleted automatically + """ + try: + try: + cred = await Credential.get(credential_id) + except ValueError as decrypt_err: + # Credential exists but can't be decrypted (wrong encryption key). + # Fall back to direct DB operations for deletion. + logger.warning( + f"Cannot decrypt credential {credential_id}, " + f"falling back to direct delete: {decrypt_err}" + ) + + # Query linked models + linked = await repo_query( + "SELECT * FROM model WHERE credential = $cred_id", + {"cred_id": ensure_record_id(credential_id)}, + ) + deleted_models = 0 + + if linked and migrate_to: + # Migrate models to another credential + target_cred = await Credential.get(migrate_to) + for model_row in linked: + model_id = str(model_row.get("id", "")) + if model_id: + await repo_query( + "UPDATE $model_id SET credential = $target_id", + { + "model_id": ensure_record_id(model_id), + "target_id": ensure_record_id(target_cred.id), + }, + ) + elif linked: + # Cascade-delete linked models + for model_row in linked: + model_id = str(model_row.get("id", "")) + if model_id: + await repo_delete(model_id) + deleted_models += 1 + + # Delete the credential itself + await repo_delete(credential_id) + + return CredentialDeleteResponse( + message="Credential deleted successfully", + deleted_models=deleted_models, + ) + + linked_models = await cred.get_linked_models() + + deleted_models = 0 + + if linked_models and migrate_to: + # Migrate models to another credential + target_cred = await Credential.get(migrate_to) + for model in linked_models: + model.credential = target_cred.id + await model.save() + + elif linked_models: + # Cascade-delete linked models (default behavior when no migrate_to) + for model in linked_models: + await model.delete() + deleted_models += 1 + + # Delete the credential + await cred.delete() + + return CredentialDeleteResponse( + message="Credential deleted successfully", + deleted_models=deleted_models, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting credential {credential_id}: {e}") + raise HTTPException(status_code=500, detail="Failed to delete credential") + + +# ============================================================================= +# Test / Discover / Register endpoints +# ============================================================================= + + +@router.post("/{credential_id}/test") +async def test_credential(credential_id: str): + """Test connection using this credential's configuration.""" + return await svc_test_credential(credential_id) + + +@router.post("/{credential_id}/discover", response_model=DiscoverModelsResponse) +async def discover_models_for_credential(credential_id: str): + """Discover available models using this credential's API key.""" + try: + cred = await Credential.get(credential_id) + config = cred.to_esperanto_config() + provider = cred.provider.lower() + + discovered = await discover_with_config(provider, config) + + return DiscoverModelsResponse( + credential_id=cred.id or "", + provider=provider, + discovered=[ + DiscoveredModelResponse( + name=d["name"], + provider=d["provider"], + description=d.get("description"), + ) + for d in discovered + ], + ) + + except Exception as e: + logger.error(f"Error discovering models for credential {credential_id}: {e}") + raise HTTPException(status_code=500, detail="Failed to discover models") + + +@router.post("/{credential_id}/register-models", response_model=RegisterModelsResponse) +async def register_models_for_credential( + credential_id: str, request: RegisterModelsRequest +): + """Register discovered models and link them to this credential.""" + try: + result = await register_models(credential_id, request.models) + return RegisterModelsResponse(**result) + except Exception as e: + logger.error(f"Error registering models for credential {credential_id}: {e}") + raise HTTPException(status_code=500, detail="Failed to register models") + + +# ============================================================================= +# Migration endpoints +# ============================================================================= + + +@router.post("/migrate-from-provider-config") +async def migrate_from_provider_config(): + """Migrate existing ProviderConfig data to individual credential records.""" + try: + return await svc_migrate_from_provider_config() + except ValueError as e: + raise _handle_value_error(e) + except Exception as e: + logger.error(f"ProviderConfig migration FAILED: {type(e).__name__}: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Migration from provider config failed") + + +@router.post("/migrate-from-env") +async def migrate_from_env(): + """Migrate API keys from environment variables to credential records.""" + try: + return await svc_migrate_from_env() + except ValueError as e: + raise _handle_value_error(e) + except Exception as e: + logger.error(f"Env migration FAILED: {type(e).__name__}: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Migration from environment variables failed") diff --git a/api/routers/embedding.py b/api/routers/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..0e6f5cb6ac51e1948b5b71eb42d898bf20734355 --- /dev/null +++ b/api/routers/embedding.py @@ -0,0 +1,113 @@ +from fastapi import APIRouter, HTTPException +from loguru import logger + +from api.command_service import CommandService +from api.models import EmbedRequest, EmbedResponse +from open_notebook.ai.models import model_manager +from open_notebook.domain.notebook import Note, Source + +router = APIRouter() + + +@router.post("/embed", response_model=EmbedResponse) +async def embed_content(embed_request: EmbedRequest): + """Embed content for vector search.""" + try: + # Check if embedding model is available + if not await model_manager.get_embedding_model(): + raise HTTPException( + status_code=400, + detail="No embedding model configured. Please configure one in the Models section.", + ) + + item_id = embed_request.item_id + item_type = embed_request.item_type.lower() + + # Validate item type + if item_type not in ["source", "note"]: + raise HTTPException( + status_code=400, detail="Item type must be either 'source' or 'note'" + ) + + # Branch based on processing mode + if embed_request.async_processing: + # ASYNC PATH: Submit command for background processing + logger.info(f"Using async processing for {item_type} {item_id}") + + try: + # Import commands to ensure they're registered + import commands.embedding_commands # noqa: F401 + + # Submit type-specific command + if item_type == "source": + command_name = "embed_source" + command_input = {"source_id": item_id} + else: # note + command_name = "embed_note" + command_input = {"note_id": item_id} + + command_id = await CommandService.submit_command_job( + "open_notebook", + command_name, + command_input, + ) + + logger.info(f"Submitted async {command_name} command: {command_id}") + + return EmbedResponse( + success=True, + message="Embedding queued for background processing", + item_id=item_id, + item_type=item_type, + command_id=command_id, + ) + + except Exception as e: + logger.error(f"Failed to submit async embedding command: {e}") + raise HTTPException( + status_code=500, detail=f"Failed to queue embedding: {str(e)}" + ) + + else: + # DOMAIN MODEL PATH: Submit job via domain model convenience methods + # These methods internally call submit_command() - still fire-and-forget + logger.info(f"Using domain model path for {item_type} {item_id}") + + command_id = None + + # Get the item and submit embedding job + if item_type == "source": + source_item = await Source.get(item_id) + if not source_item: + raise HTTPException(status_code=404, detail="Source not found") + + # Submit embed_source job (returns command_id for tracking) + command_id = await source_item.vectorize() + message = "Source embedding job submitted" + + elif item_type == "note": + note_item = await Note.get(item_id) + if not note_item: + raise HTTPException(status_code=404, detail="Note not found") + + # Note.save() internally submits embed_note command and returns command_id + command_id = await note_item.save() + message = "Note embedding job submitted" + + return EmbedResponse( + success=True, + message=message, + item_id=item_id, + item_type=item_type, + command_id=command_id, + ) + + except HTTPException: + raise + except Exception as e: + logger.error( + f"Error embedding {embed_request.item_type} {embed_request.item_id}: {str(e)}" + ) + raise HTTPException( + status_code=500, detail=f"Error embedding content: {str(e)}" + ) diff --git a/api/routers/embedding_rebuild.py b/api/routers/embedding_rebuild.py new file mode 100644 index 0000000000000000000000000000000000000000..155d8c0fe185a28cb6a6dd8b636aab82680a8f7f --- /dev/null +++ b/api/routers/embedding_rebuild.py @@ -0,0 +1,192 @@ +from fastapi import APIRouter, HTTPException +from loguru import logger +from surreal_commands import get_command_status + +from api.command_service import CommandService +from api.models import ( + RebuildProgress, + RebuildRequest, + RebuildResponse, + RebuildStats, + RebuildStatusResponse, +) +from open_notebook.database.repository import repo_query + +router = APIRouter() + + +@router.post("/rebuild", response_model=RebuildResponse) +async def start_rebuild(request: RebuildRequest): + """ + Start a background job to rebuild embeddings. + + - **mode**: "existing" (re-embed items with embeddings) or "all" (embed everything) + - **include_sources**: Include sources in rebuild (default: true) + - **include_notes**: Include notes in rebuild (default: true) + - **include_insights**: Include insights in rebuild (default: true) + + Returns command ID to track progress and estimated item count. + """ + try: + logger.info(f"Starting rebuild request: mode={request.mode}") + + # Import commands to ensure they're registered + import commands.embedding_commands # noqa: F401 + + # Estimate total items (quick count query) + # This is a rough estimate before the command runs + total_estimate = 0 + + if request.include_sources: + if request.mode == "existing": + # Count sources with embeddings + result = await repo_query( + """ + SELECT VALUE count(array::distinct( + SELECT VALUE source.id + FROM source_embedding + WHERE embedding != none AND array::len(embedding) > 0 + )) as count FROM {} + """ + ) + else: + # Count all sources with content + result = await repo_query( + "SELECT VALUE count() as count FROM source WHERE full_text != none GROUP ALL" + ) + + if result and isinstance(result[0], dict): + total_estimate += result[0].get("count", 0) + elif result: + total_estimate += result[0] if isinstance(result[0], int) else 0 + + if request.include_notes: + if request.mode == "existing": + result = await repo_query( + "SELECT VALUE count() as count FROM note WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL" + ) + else: + result = await repo_query( + "SELECT VALUE count() as count FROM note WHERE content != none GROUP ALL" + ) + + if result and isinstance(result[0], dict): + total_estimate += result[0].get("count", 0) + elif result: + total_estimate += result[0] if isinstance(result[0], int) else 0 + + if request.include_insights: + if request.mode == "existing": + result = await repo_query( + "SELECT VALUE count() as count FROM source_insight WHERE embedding != none AND array::len(embedding) > 0 GROUP ALL" + ) + else: + result = await repo_query( + "SELECT VALUE count() as count FROM source_insight GROUP ALL" + ) + + if result and isinstance(result[0], dict): + total_estimate += result[0].get("count", 0) + elif result: + total_estimate += result[0] if isinstance(result[0], int) else 0 + + logger.info(f"Estimated {total_estimate} items to process") + + # Submit command + command_id = await CommandService.submit_command_job( + "open_notebook", + "rebuild_embeddings", + { + "mode": request.mode, + "include_sources": request.include_sources, + "include_notes": request.include_notes, + "include_insights": request.include_insights, + }, + ) + + logger.info(f"Submitted rebuild command: {command_id}") + + return RebuildResponse( + command_id=command_id, + total_items=total_estimate, + message=f"Rebuild operation started. Estimated {total_estimate} items to process.", + ) + + except Exception as e: + logger.error(f"Failed to start rebuild: {e}") + logger.exception(e) + raise HTTPException( + status_code=500, detail=f"Failed to start rebuild operation: {str(e)}" + ) + + +@router.get("/rebuild/{command_id}/status", response_model=RebuildStatusResponse) +async def get_rebuild_status(command_id: str): + """ + Get the status of a rebuild operation. + + Returns: + - **status**: queued, running, completed, failed + - **progress**: processed count, total count, percentage + - **stats**: breakdown by type (sources, notes, insights, failed) + - **timestamps**: started_at, completed_at + """ + try: + # Get command status from surreal_commands + status = await get_command_status(command_id) + + if not status: + raise HTTPException(status_code=404, detail="Rebuild command not found") + + # Build response based on status + response = RebuildStatusResponse( + command_id=command_id, + status=status.status, + ) + + # Extract metadata from command result + if status.result and isinstance(status.result, dict): + result = status.result + + # Build progress info + if "total_items" in result and "jobs_submitted" in result: + total = result["total_items"] + submitted = result["jobs_submitted"] + response.progress = RebuildProgress( + processed=submitted, + total=total, + percentage=round((submitted / total * 100) if total > 0 else 0, 2), + ) + + # Build stats + response.stats = RebuildStats( + sources=result.get("sources_submitted", 0), + notes=result.get("notes_submitted", 0), + insights=result.get("insights_submitted", 0), + failed=result.get("failed_submissions", 0), + ) + + # Add timestamps + if hasattr(status, "created") and status.created: + response.started_at = str(status.created) + if hasattr(status, "updated") and status.updated: + response.completed_at = str(status.updated) + + # Add error message if failed + if ( + status.status == "failed" + and status.result + and isinstance(status.result, dict) + ): + response.error_message = status.result.get("error_message", "Unknown error") + + return response + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get rebuild status: {e}") + logger.exception(e) + raise HTTPException( + status_code=500, detail=f"Failed to get rebuild status: {str(e)}" + ) diff --git a/api/routers/episode_profiles.py b/api/routers/episode_profiles.py new file mode 100644 index 0000000000000000000000000000000000000000..cc5e0708334a601bb61c3a679883b852c16c2aa2 --- /dev/null +++ b/api/routers/episode_profiles.py @@ -0,0 +1,226 @@ +from typing import List, Optional + +from fastapi import APIRouter, HTTPException +from loguru import logger +from pydantic import BaseModel, Field + +from open_notebook.podcasts.models import EpisodeProfile + +router = APIRouter() + + +class EpisodeProfileResponse(BaseModel): + id: str + name: str + description: str + speaker_config: str + outline_llm: Optional[str] = None + transcript_llm: Optional[str] = None + language: Optional[str] = None + default_briefing: str + num_segments: int + # Legacy fields (for display/migration awareness) + outline_provider: Optional[str] = None + outline_model: Optional[str] = None + transcript_provider: Optional[str] = None + transcript_model: Optional[str] = None + + +def _profile_to_response(profile: EpisodeProfile) -> EpisodeProfileResponse: + return EpisodeProfileResponse( + id=str(profile.id), + name=profile.name, + description=profile.description or "", + speaker_config=profile.speaker_config, + outline_llm=profile.outline_llm, + transcript_llm=profile.transcript_llm, + language=profile.language, + default_briefing=profile.default_briefing, + num_segments=profile.num_segments, + outline_provider=profile.outline_provider, + outline_model=profile.outline_model, + transcript_provider=profile.transcript_provider, + transcript_model=profile.transcript_model, + ) + + +@router.get("/episode-profiles", response_model=List[EpisodeProfileResponse]) +async def list_episode_profiles(): + """List all available episode profiles""" + try: + profiles = await EpisodeProfile.get_all(order_by="name asc") + return [_profile_to_response(p) for p in profiles] + except Exception as e: + logger.error(f"Failed to fetch episode profiles: {e}") + raise HTTPException( + status_code=500, detail="Failed to fetch episode profiles" + ) + + +@router.get("/episode-profiles/{profile_name}", response_model=EpisodeProfileResponse) +async def get_episode_profile(profile_name: str): + """Get a specific episode profile by name""" + try: + profile = await EpisodeProfile.get_by_name(profile_name) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Episode profile '{profile_name}' not found" + ) + + return _profile_to_response(profile) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to fetch episode profile '{profile_name}': {e}") + raise HTTPException( + status_code=500, detail="Failed to fetch episode profile" + ) + + +class EpisodeProfileCreate(BaseModel): + name: str = Field(..., description="Unique profile name") + description: str = Field("", description="Profile description") + speaker_config: str = Field(..., description="Reference to speaker profile name") + outline_llm: Optional[str] = Field(None, description="Model record ID for outline") + transcript_llm: Optional[str] = Field( + None, description="Model record ID for transcript" + ) + language: Optional[str] = Field(None, description="Podcast language code") + default_briefing: str = Field(..., description="Default briefing template") + num_segments: int = Field(default=5, description="Number of podcast segments") + # Legacy fields (accepted but not required) + outline_provider: Optional[str] = None + outline_model: Optional[str] = None + transcript_provider: Optional[str] = None + transcript_model: Optional[str] = None + + +@router.post("/episode-profiles", response_model=EpisodeProfileResponse) +async def create_episode_profile(profile_data: EpisodeProfileCreate): + """Create a new episode profile""" + try: + profile = EpisodeProfile( + name=profile_data.name, + description=profile_data.description, + speaker_config=profile_data.speaker_config, + outline_llm=profile_data.outline_llm, + transcript_llm=profile_data.transcript_llm, + language=profile_data.language, + default_briefing=profile_data.default_briefing, + num_segments=profile_data.num_segments, + outline_provider=profile_data.outline_provider, + outline_model=profile_data.outline_model, + transcript_provider=profile_data.transcript_provider, + transcript_model=profile_data.transcript_model, + ) + + await profile.save() + return _profile_to_response(profile) + + except Exception as e: + logger.error(f"Failed to create episode profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to create episode profile" + ) + + +@router.put("/episode-profiles/{profile_id}", response_model=EpisodeProfileResponse) +async def update_episode_profile(profile_id: str, profile_data: EpisodeProfileCreate): + """Update an existing episode profile""" + try: + profile = await EpisodeProfile.get(profile_id) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Episode profile '{profile_id}' not found" + ) + + profile.name = profile_data.name + profile.description = profile_data.description + profile.speaker_config = profile_data.speaker_config + profile.outline_llm = profile_data.outline_llm + profile.transcript_llm = profile_data.transcript_llm + profile.language = profile_data.language + profile.default_briefing = profile_data.default_briefing + profile.num_segments = profile_data.num_segments + profile.outline_provider = profile_data.outline_provider + profile.outline_model = profile_data.outline_model + profile.transcript_provider = profile_data.transcript_provider + profile.transcript_model = profile_data.transcript_model + + await profile.save() + return _profile_to_response(profile) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update episode profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to update episode profile" + ) + + +@router.delete("/episode-profiles/{profile_id}") +async def delete_episode_profile(profile_id: str): + """Delete an episode profile""" + try: + profile = await EpisodeProfile.get(profile_id) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Episode profile '{profile_id}' not found" + ) + + await profile.delete() + + return {"message": "Episode profile deleted successfully"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete episode profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to delete episode profile" + ) + + +@router.post( + "/episode-profiles/{profile_id}/duplicate", response_model=EpisodeProfileResponse +) +async def duplicate_episode_profile(profile_id: str): + """Duplicate an episode profile""" + try: + original = await EpisodeProfile.get(profile_id) + + if not original: + raise HTTPException( + status_code=404, detail=f"Episode profile '{profile_id}' not found" + ) + + duplicate = EpisodeProfile( + name=f"{original.name} - Copy", + description=original.description, + speaker_config=original.speaker_config, + outline_llm=original.outline_llm, + transcript_llm=original.transcript_llm, + language=original.language, + default_briefing=original.default_briefing, + num_segments=original.num_segments, + outline_provider=original.outline_provider, + outline_model=original.outline_model, + transcript_provider=original.transcript_provider, + transcript_model=original.transcript_model, + ) + + await duplicate.save() + return _profile_to_response(duplicate) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to duplicate episode profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to duplicate episode profile" + ) diff --git a/api/routers/insights.py b/api/routers/insights.py new file mode 100644 index 0000000000000000000000000000000000000000..b9e2c7145fcc3428a1fb68fdf5bfe4bc3883bf8d --- /dev/null +++ b/api/routers/insights.py @@ -0,0 +1,82 @@ +from fastapi import APIRouter, HTTPException +from loguru import logger + +from api.models import NoteResponse, SaveAsNoteRequest, SourceInsightResponse +from open_notebook.domain.notebook import SourceInsight +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +@router.get("/insights/{insight_id}", response_model=SourceInsightResponse) +async def get_insight(insight_id: str): + """Get a specific insight by ID.""" + try: + insight = await SourceInsight.get(insight_id) + if not insight: + raise HTTPException(status_code=404, detail="Insight not found") + + # Get source ID from the insight relationship + source = await insight.get_source() + + return SourceInsightResponse( + id=insight.id or "", + source_id=source.id or "", + insight_type=insight.insight_type, + content=insight.content, + created=str(insight.created), + updated=str(insight.updated), + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching insight {insight_id}: {str(e)}") + raise HTTPException(status_code=500, detail="Error fetching insight") + + +@router.delete("/insights/{insight_id}") +async def delete_insight(insight_id: str): + """Delete a specific insight.""" + try: + insight = await SourceInsight.get(insight_id) + if not insight: + raise HTTPException(status_code=404, detail="Insight not found") + + await insight.delete() + + return {"message": "Insight deleted successfully"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting insight {insight_id}: {str(e)}") + raise HTTPException(status_code=500, detail="Error deleting insight") + + +@router.post("/insights/{insight_id}/save-as-note", response_model=NoteResponse) +async def save_insight_as_note(insight_id: str, request: SaveAsNoteRequest): + """Convert an insight to a note.""" + try: + insight = await SourceInsight.get(insight_id) + if not insight: + raise HTTPException(status_code=404, detail="Insight not found") + + # Use the existing save_as_note method from the domain model + note = await insight.save_as_note(request.notebook_id) + + return NoteResponse( + id=note.id or "", + title=note.title, + content=note.content, + note_type=note.note_type, + created=str(note.created), + updated=str(note.updated), + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error saving insight {insight_id} as note: {str(e)}") + raise HTTPException( + status_code=500, detail="Error saving insight as note" + ) diff --git a/api/routers/languages.py b/api/routers/languages.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d1db88c56301f0d7844caf7ea50f6921902040 --- /dev/null +++ b/api/routers/languages.py @@ -0,0 +1,83 @@ +from typing import List + +import pycountry +from babel import Locale +from babel.core import get_global +from fastapi import APIRouter +from pydantic import BaseModel + +router = APIRouter() + +# Additional regional variants for languages where the distinction matters +# (TTS accent, vocabulary, spelling differences) +_EXTRA_VARIANTS = [ + "pt_PT", + "en_GB", + "en_AU", + "en_IN", + "es_MX", + "es_AR", + "es_CO", + "fr_CA", + "fr_CH", + "zh_TW", + "zh_HK", + "de_AT", + "de_CH", + "ar_SA", + "nl_BE", +] + + +class LanguageResponse(BaseModel): + code: str + name: str + + +@router.get("/languages", response_model=List[LanguageResponse]) +async def list_languages(): + """List available languages as BCP 47 locale codes (e.g. pt-BR, en-US).""" + likely_subtags = get_global("likely_subtags") + languages = [] + seen = set() + + # 1. For each language, resolve its default locale via CLDR likely subtags + for lang in pycountry.languages: + if not hasattr(lang, "alpha_2"): + continue + + code = lang.alpha_2 + likely = likely_subtags.get(code) + + if likely: + try: + loc = Locale.parse(likely) + if loc.territory: + bcp47 = f"{loc.language}-{loc.territory}" + display = loc.get_display_name("en") + if bcp47 not in seen: + seen.add(bcp47) + languages.append(LanguageResponse(code=bcp47, name=display)) + continue + except Exception: + pass + + # Fallback: bare language code + if code not in seen: + seen.add(code) + languages.append(LanguageResponse(code=code, name=lang.name)) + + # 2. Add important regional variants + for locale_str in _EXTRA_VARIANTS: + try: + loc = Locale.parse(locale_str) + bcp47 = f"{loc.language}-{loc.territory}" + if bcp47 not in seen: + seen.add(bcp47) + display = loc.get_display_name("en") + languages.append(LanguageResponse(code=bcp47, name=display)) + except Exception: + pass + + languages.sort(key=lambda x: x.name) + return languages diff --git a/api/routers/models.py b/api/routers/models.py new file mode 100644 index 0000000000000000000000000000000000000000..df28463bf6ad5b87e252630ceb7a45ce94a9465c --- /dev/null +++ b/api/routers/models.py @@ -0,0 +1,776 @@ +import os +import traceback +from typing import Dict, List, Optional + +from esperanto import AIFactory +from fastapi import APIRouter, HTTPException, Query +from loguru import logger +from pydantic import BaseModel + +from api.models import ( + DefaultModelsResponse, + ModelCreate, + ModelResponse, + ProviderAvailabilityResponse, +) +from open_notebook.domain.credential import Credential +from open_notebook.ai.connection_tester import test_individual_model +from open_notebook.ai.key_provider import provision_provider_keys +from open_notebook.ai.model_discovery import ( + discover_provider_models, + get_provider_model_count, + sync_all_providers, + sync_provider_models, +) +from open_notebook.ai.models import DefaultModels, Model +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +# ============================================================================= +# Model Discovery Response Models +# ============================================================================= + + +class DiscoveredModelResponse(BaseModel): + """Response model for a discovered model.""" + + name: str + provider: str + model_type: str + description: Optional[str] = None + + +class ProviderSyncResponse(BaseModel): + """Response model for provider sync operation.""" + + provider: str + discovered: int + new: int + existing: int + + +class AllProvidersSyncResponse(BaseModel): + """Response model for syncing all providers.""" + + results: Dict[str, ProviderSyncResponse] + total_discovered: int + total_new: int + + +class ProviderModelCountResponse(BaseModel): + """Response model for provider model counts.""" + + provider: str + counts: Dict[str, int] + total: int + + +class AutoAssignResult(BaseModel): + """Response model for auto-assign operation.""" + + assigned: Dict[str, str] # slot_name -> model_id + skipped: List[str] # slots already assigned + missing: List[str] # slots with no available models + + +class ModelTestResponse(BaseModel): + """Response model for individual model test.""" + + success: bool + message: str + details: Optional[str] = None + + +# Provider priority for auto-assignment (higher priority first) +PROVIDER_PRIORITY = [ + "openai", + "anthropic", + "google", + "mistral", + "groq", + "deepseek", + "xai", + "openrouter", + "ollama", + "azure", + "openai_compatible", + "dashscope", + "minimax", +] + +# Model preference patterns (preferred models within each provider) +MODEL_PREFERENCES = { + "openai": ["gpt-4o", "gpt-4", "gpt-3.5-turbo"], + "anthropic": ["claude-3-5-sonnet", "claude-3-opus", "claude-3-sonnet"], + "google": ["gemini-2.0", "gemini-1.5-pro", "gemini-pro"], + "mistral": ["mistral-large", "mixtral"], + "groq": ["llama-3.3", "llama-3.1", "mixtral"], + "dashscope": ["qwen-max", "qwen-plus", "qwen-turbo"], + "minimax": ["MiniMax-M2.5", "MiniMax-M2.5-highspeed"], +} + + +async def _check_provider_has_credential(provider: str) -> bool: + """Check if a provider has any credentials configured in the database.""" + try: + credentials = await Credential.get_by_provider(provider) + return len(credentials) > 0 + except Exception: + pass + return False + + +def _check_azure_support(mode: str) -> bool: + """ + Check if Azure OpenAI provider is available for a specific mode. + + Args: + mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS' + + Returns: + bool: True if either generic or mode-specific env vars are set + """ + # Check generic configuration (applies to all modes) + generic = ( + os.environ.get("AZURE_OPENAI_API_KEY") is not None + and os.environ.get("AZURE_OPENAI_ENDPOINT") is not None + and os.environ.get("AZURE_OPENAI_API_VERSION") is not None + ) + + # Check mode-specific configuration (takes precedence) + specific = ( + os.environ.get(f"AZURE_OPENAI_API_KEY_{mode}") is not None + and os.environ.get(f"AZURE_OPENAI_ENDPOINT_{mode}") is not None + and os.environ.get(f"AZURE_OPENAI_API_VERSION_{mode}") is not None + ) + + return generic or specific + + +def _check_openai_compatible_support(mode: str) -> bool: + """ + Check if OpenAI-compatible provider is available for a specific mode. + + Args: + mode: One of 'LLM', 'EMBEDDING', 'STT', 'TTS' + + Returns: + bool: True if either generic or mode-specific env var is set + """ + generic = os.environ.get("OPENAI_COMPATIBLE_BASE_URL") is not None + specific = os.environ.get(f"OPENAI_COMPATIBLE_BASE_URL_{mode}") is not None + generic_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") is not None + specific_key = os.environ.get(f"OPENAI_COMPATIBLE_API_KEY_{mode}") is not None + return generic or specific or generic_key or specific_key + + +@router.get("/models", response_model=List[ModelResponse]) +async def get_models( + type: Optional[str] = Query(None, description="Filter by model type"), +): + """Get all configured models with optional type filtering.""" + try: + if type: + models = await Model.get_models_by_type(type) + else: + models = await Model.get_all() + + return [ + ModelResponse( + id=model.id, + name=model.name, + provider=model.provider, + type=model.type, + credential=model.credential, + created=str(model.created), + updated=str(model.updated), + ) + for model in models + ] + except Exception as e: + logger.error(f"Error fetching models: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching models: {str(e)}") + + +@router.post("/models", response_model=ModelResponse) +async def create_model(model_data: ModelCreate): + """Create a new model configuration.""" + try: + # Validate model type + valid_types = ["language", "embedding", "text_to_speech", "speech_to_text"] + if model_data.type not in valid_types: + raise HTTPException( + status_code=400, + detail=f"Invalid model type. Must be one of: {valid_types}", + ) + + # Check for duplicate model name under the same provider and type (case-insensitive) + from open_notebook.database.repository import repo_query + + existing = await repo_query( + "SELECT * FROM model WHERE string::lowercase(provider) = $provider AND string::lowercase(name) = $name AND string::lowercase(type) = $type LIMIT 1", + { + "provider": model_data.provider.lower(), + "name": model_data.name.lower(), + "type": model_data.type.lower(), + }, + ) + if existing: + raise HTTPException( + status_code=400, + detail=f"Model '{model_data.name}' already exists for provider '{model_data.provider}' with type '{model_data.type}'", + ) + + new_model = Model( + name=model_data.name, + provider=model_data.provider, + type=model_data.type, + credential=model_data.credential, + ) + await new_model.save() + + return ModelResponse( + id=new_model.id or "", + name=new_model.name, + provider=new_model.provider, + type=new_model.type, + credential=new_model.credential, + created=str(new_model.created), + updated=str(new_model.updated), + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error creating model: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error creating model: {str(e)}") + + +@router.delete("/models/{model_id}") +async def delete_model(model_id: str): + """Delete a model configuration.""" + try: + model = await Model.get(model_id) + if not model: + raise HTTPException(status_code=404, detail="Model not found") + + await model.delete() + + return {"message": "Model deleted successfully"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting model {model_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error deleting model: {str(e)}") + + +@router.post("/models/{model_id}/test", response_model=ModelTestResponse) +async def test_model(model_id: str): + """Test if a specific model is correctly configured and functional.""" + try: + model = await Model.get(model_id) + if not model: + raise HTTPException(status_code=404, detail="Model not found") + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=404, detail="Model not found") + + try: + success, message = await test_individual_model(model) + return ModelTestResponse(success=success, message=message) + except Exception as e: + logger.error(f"Error testing model {model_id}: {traceback.format_exc()}") + return ModelTestResponse( + success=False, + message=str(e)[:200], + ) + + +@router.get("/models/defaults", response_model=DefaultModelsResponse) +async def get_default_models(): + """Get default model assignments.""" + try: + defaults = await DefaultModels.get_instance() + + return DefaultModelsResponse( + default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined] + default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined] + large_context_model=defaults.large_context_model, # type: ignore[attr-defined] + default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined] + default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined] + default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined] + default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined] + ) + except Exception as e: + logger.error(f"Error fetching default models: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching default models: {str(e)}" + ) + + +@router.put("/models/defaults", response_model=DefaultModelsResponse) +async def update_default_models(defaults_data: DefaultModelsResponse): + """Update default model assignments.""" + try: + defaults = await DefaultModels.get_instance() + + # Update only provided fields + if defaults_data.default_chat_model is not None: + defaults.default_chat_model = defaults_data.default_chat_model # type: ignore[attr-defined] + if defaults_data.default_transformation_model is not None: + defaults.default_transformation_model = ( + defaults_data.default_transformation_model + ) # type: ignore[attr-defined] + if defaults_data.large_context_model is not None: + defaults.large_context_model = defaults_data.large_context_model # type: ignore[attr-defined] + if defaults_data.default_text_to_speech_model is not None: + defaults.default_text_to_speech_model = ( + defaults_data.default_text_to_speech_model + ) # type: ignore[attr-defined] + if defaults_data.default_speech_to_text_model is not None: + defaults.default_speech_to_text_model = ( + defaults_data.default_speech_to_text_model + ) # type: ignore[attr-defined] + if defaults_data.default_embedding_model is not None: + defaults.default_embedding_model = defaults_data.default_embedding_model # type: ignore[attr-defined] + if defaults_data.default_tools_model is not None: + defaults.default_tools_model = defaults_data.default_tools_model # type: ignore[attr-defined] + + await defaults.update() + + # No cache refresh needed - next access will fetch fresh data from DB + + return DefaultModelsResponse( + default_chat_model=defaults.default_chat_model, # type: ignore[attr-defined] + default_transformation_model=defaults.default_transformation_model, # type: ignore[attr-defined] + large_context_model=defaults.large_context_model, # type: ignore[attr-defined] + default_text_to_speech_model=defaults.default_text_to_speech_model, # type: ignore[attr-defined] + default_speech_to_text_model=defaults.default_speech_to_text_model, # type: ignore[attr-defined] + default_embedding_model=defaults.default_embedding_model, # type: ignore[attr-defined] + default_tools_model=defaults.default_tools_model, # type: ignore[attr-defined] + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating default models: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error updating default models: {str(e)}" + ) + + +@router.get("/models/providers", response_model=ProviderAvailabilityResponse) +async def get_provider_availability(): + """Get provider availability based on database config and environment variables.""" + try: + # Check which providers have credentials in the database or env vars + # For each provider, check DB credentials first, then env vars as fallback + + # Simple env var mapping for backward compatibility + env_var_map = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "google": "GOOGLE_API_KEY", + "groq": "GROQ_API_KEY", + "mistral": "MISTRAL_API_KEY", + "deepseek": "DEEPSEEK_API_KEY", + "xai": "XAI_API_KEY", + "openrouter": "OPENROUTER_API_KEY", + "voyage": "VOYAGE_API_KEY", + "elevenlabs": "ELEVENLABS_API_KEY", + "ollama": "OLLAMA_API_BASE", + "dashscope": "DASHSCOPE_API_KEY", + "minimax": "MINIMAX_API_KEY", + } + + provider_status = {} + + # Check simple providers: credential in DB or env var + for provider, env_var in env_var_map.items(): + has_cred = await _check_provider_has_credential(provider) + has_env = os.environ.get(env_var) is not None + provider_status[provider] = has_cred or has_env + + # Google also supports GEMINI_API_KEY + if not provider_status.get("google"): + provider_status["google"] = os.environ.get("GEMINI_API_KEY") is not None + + # Vertex: DB credential or env vars + provider_status["vertex"] = ( + await _check_provider_has_credential("vertex") + or os.environ.get("VERTEX_PROJECT") is not None + ) + + # Azure: DB credential or env vars + provider_status["azure"] = ( + await _check_provider_has_credential("azure") + or _check_azure_support("LLM") + or _check_azure_support("EMBEDDING") + or _check_azure_support("STT") + or _check_azure_support("TTS") + ) + + # OpenAI-compatible: DB credential or env vars + provider_status["openai-compatible"] = ( + await _check_provider_has_credential("openai_compatible") + or _check_openai_compatible_support("LLM") + or _check_openai_compatible_support("EMBEDDING") + or _check_openai_compatible_support("STT") + or _check_openai_compatible_support("TTS") + ) + + available_providers = [k for k, v in provider_status.items() if v] + unavailable_providers = [k for k, v in provider_status.items() if not v] + + # Get supported model types from Esperanto + esperanto_available = AIFactory.get_available_providers() + + # Build supported types mapping only for available providers + supported_types: dict[str, list[str]] = {} + for provider in available_providers: + supported_types[provider] = [] + + # Map Esperanto model types to our environment variable modes + mode_mapping = { + "language": "LLM", + "embedding": "EMBEDDING", + "speech_to_text": "STT", + "text_to_speech": "TTS", + } + + # Special handling for openai-compatible to check mode-specific availability + if provider == "openai-compatible": + has_db_cred = await _check_provider_has_credential("openai_compatible") + for model_type, mode in mode_mapping.items(): + if ( + model_type in esperanto_available + and provider in esperanto_available[model_type] + ): + if has_db_cred or _check_openai_compatible_support(mode): + supported_types[provider].append(model_type) + # Special handling for azure to check mode-specific availability + elif provider == "azure": + has_db_cred = await _check_provider_has_credential("azure") + for model_type, mode in mode_mapping.items(): + if ( + model_type in esperanto_available + and provider in esperanto_available[model_type] + ): + if has_db_cred or _check_azure_support(mode): + supported_types[provider].append(model_type) + else: + # Standard provider detection + for model_type, providers in esperanto_available.items(): + if provider in providers: + supported_types[provider].append(model_type) + + return ProviderAvailabilityResponse( + available=available_providers, + unavailable=unavailable_providers, + supported_types=supported_types, + ) + except Exception as e: + logger.error(f"Error checking provider availability: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error checking provider availability: {str(e)}" + ) + + +# ============================================================================= +# Model Discovery Endpoints +# ============================================================================= + + +@router.get( + "/models/discover/{provider}", response_model=List[DiscoveredModelResponse] +) +async def discover_models(provider: str): + """ + Discover available models from a provider without registering them. + + This endpoint queries the provider's API to list available models + but does not save them to the database. Use the sync endpoint + to both discover and register models. + """ + try: + # Provision DB-stored credentials into env vars before discovery + await provision_provider_keys(provider) + discovered = await discover_provider_models(provider) + return [ + DiscoveredModelResponse( + name=m.name, + provider=m.provider, + model_type=m.model_type, + description=m.description, + ) + for m in discovered + ] + except Exception as e: + logger.error(f"Error discovering models for {provider}: {str(e)}") + raise HTTPException( + status_code=500, detail="Error discovering models. Check server logs for details." + ) + + +@router.post("/models/sync/{provider}", response_model=ProviderSyncResponse) +async def sync_models(provider: str): + """ + Sync models for a specific provider. + + Discovers available models from the provider's API and registers + any new models in the database. Existing models are skipped. + + Returns counts of discovered, new, and existing models. + """ + try: + # Provision DB-stored credentials into env vars before discovery + await provision_provider_keys(provider) + discovered, new, existing = await sync_provider_models( + provider, auto_register=True + ) + return ProviderSyncResponse( + provider=provider, + discovered=discovered, + new=new, + existing=existing, + ) + except Exception as e: + logger.error(f"Error syncing models for {provider}: {str(e)}") + raise HTTPException(status_code=500, detail="Error syncing models. Check server logs for details.") + + +@router.post("/models/sync", response_model=AllProvidersSyncResponse) +async def sync_all_models(): + """ + Sync models for all configured providers. + + Discovers and registers models from all providers that have + valid API keys configured. This is useful for initial setup + or periodic refresh of available models. + """ + try: + results = await sync_all_providers() + + response_results = {} + total_discovered = 0 + total_new = 0 + + for provider, (discovered, new, existing) in results.items(): + response_results[provider] = ProviderSyncResponse( + provider=provider, + discovered=discovered, + new=new, + existing=existing, + ) + total_discovered += discovered + total_new += new + + return AllProvidersSyncResponse( + results=response_results, + total_discovered=total_discovered, + total_new=total_new, + ) + except Exception as e: + logger.error(f"Error syncing all models: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error syncing all models: {str(e)}" + ) + + +@router.get("/models/count/{provider}", response_model=ProviderModelCountResponse) +async def get_model_count(provider: str): + """ + Get count of registered models for a provider, grouped by type. + + Returns counts for each model type (language, embedding, + speech_to_text, text_to_speech) as well as total count. + """ + try: + counts = await get_provider_model_count(provider) + total = sum(counts.values()) + return ProviderModelCountResponse( + provider=provider, + counts=counts, + total=total, + ) + except Exception as e: + logger.error(f"Error getting model count for {provider}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error getting model count: {str(e)}" + ) + + +@router.get("/models/by-provider/{provider}", response_model=List[ModelResponse]) +async def get_models_by_provider(provider: str): + """ + Get all registered models for a specific provider. + + Returns models from the database that belong to the specified provider. + """ + try: + from open_notebook.database.repository import repo_query + + models = await repo_query( + "SELECT * FROM model WHERE provider = $provider ORDER BY type, name", + {"provider": provider}, + ) + + return [ + ModelResponse( + id=model.get("id", ""), + name=model.get("name", ""), + provider=model.get("provider", ""), + type=model.get("type", ""), + credential=model.get("credential"), + created=str(model.get("created", "")), + updated=str(model.get("updated", "")), + ) + for model in models + ] + except Exception as e: + logger.error(f"Error fetching models for {provider}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching models: {str(e)}" + ) + + +def _get_preferred_model( + models: List[Dict], provider_priority: List[str], model_preferences: Dict +) -> Optional[Dict]: + """ + Select the best model from a list based on provider priority and model preferences. + + Args: + models: List of model dictionaries with 'provider', 'name', 'id' keys + provider_priority: List of providers in preference order + model_preferences: Dict mapping provider to list of preferred model name patterns + + Returns: + The best model dict, or None if no models available + """ + if not models: + return None + + # Group models by provider + by_provider: Dict[str, List[Dict]] = {} + for model in models: + provider = model.get("provider", "") + if provider not in by_provider: + by_provider[provider] = [] + by_provider[provider].append(model) + + # Find first provider with models (in priority order) + for provider in provider_priority: + if provider in by_provider: + provider_models = by_provider[provider] + + # Check for preferred models within this provider + if provider in model_preferences: + for preference in model_preferences[provider]: + for model in provider_models: + if preference.lower() in model.get("name", "").lower(): + return model + + # Fall back to first model from this provider + return provider_models[0] + + # Fall back to first model from any provider + return models[0] if models else None + + +@router.post("/models/auto-assign", response_model=AutoAssignResult) +async def auto_assign_defaults(): + """ + Auto-assign default models based on available models. + + This endpoint intelligently assigns the first available model of each + required type to the corresponding default slot. It uses provider + priority (preferring premium providers like OpenAI, Anthropic) and + model preferences within each provider. + + Returns: + - assigned: Dict of slot names to assigned model IDs + - skipped: List of slots that already have models assigned + - missing: List of slots with no available models + """ + try: + from open_notebook.database.repository import repo_query + + # Get current defaults + defaults = await DefaultModels.get_instance() + + # Get all models grouped by type + all_models = await repo_query( + "SELECT * FROM model ORDER BY provider, name", + {}, + ) + + # Group models by type + models_by_type: Dict[str, List[Dict]] = { + "language": [], + "embedding": [], + "text_to_speech": [], + "speech_to_text": [], + } + + for model in all_models: + model_type = model.get("type", "") + if model_type in models_by_type: + models_by_type[model_type].append(model) + + # Define slot configuration: (slot_name, model_type, current_value) + slot_configs = [ + ("default_chat_model", "language", defaults.default_chat_model), # type: ignore[attr-defined] + ("default_transformation_model", "language", defaults.default_transformation_model), # type: ignore[attr-defined] + ("default_tools_model", "language", defaults.default_tools_model), # type: ignore[attr-defined] + ("large_context_model", "language", defaults.large_context_model), # type: ignore[attr-defined] + ("default_embedding_model", "embedding", defaults.default_embedding_model), # type: ignore[attr-defined] + ("default_text_to_speech_model", "text_to_speech", defaults.default_text_to_speech_model), # type: ignore[attr-defined] + ("default_speech_to_text_model", "speech_to_text", defaults.default_speech_to_text_model), # type: ignore[attr-defined] + ] + + assigned: Dict[str, str] = {} + skipped: List[str] = [] + missing: List[str] = [] + + for slot_name, model_type, current_value in slot_configs: + if current_value: + # Slot already has a value + skipped.append(slot_name) + continue + + available_models = models_by_type.get(model_type, []) + if not available_models: + # No models of this type available + missing.append(slot_name) + continue + + # Select best model for this slot + best_model = _get_preferred_model( + available_models, PROVIDER_PRIORITY, MODEL_PREFERENCES + ) + + if best_model: + model_id = best_model.get("id", "") + assigned[slot_name] = model_id + # Update the defaults object + setattr(defaults, slot_name, model_id) + + # Save updated defaults if any assignments were made + if assigned: + await defaults.update() + + return AutoAssignResult( + assigned=assigned, + skipped=skipped, + missing=missing, + ) + + except Exception as e: + logger.error(f"Error auto-assigning defaults: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error auto-assigning defaults: {str(e)}" + ) diff --git a/api/routers/notebooks.py b/api/routers/notebooks.py new file mode 100644 index 0000000000000000000000000000000000000000..fd08e44bab015ff30f450a2238100f2f0888e60b --- /dev/null +++ b/api/routers/notebooks.py @@ -0,0 +1,354 @@ +from typing import List, Optional + +from fastapi import APIRouter, HTTPException, Query +from loguru import logger + +from api.models import ( + NotebookCreate, + NotebookDeletePreview, + NotebookDeleteResponse, + NotebookResponse, + NotebookUpdate, +) +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.notebook import Notebook, Source +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +@router.get("/notebooks", response_model=List[NotebookResponse]) +async def get_notebooks( + archived: Optional[bool] = Query(None, description="Filter by archived status"), + order_by: str = Query("updated desc", description="Order by field and direction"), +): + """Get all notebooks with optional filtering and ordering.""" + try: + # Validate order_by against allowlist to prevent SurrealQL injection + allowed_fields = {"name", "created", "updated"} + allowed_directions = {"asc", "desc"} + + parts = order_by.strip().lower().split() + if len(parts) == 1: + if parts[0] not in allowed_fields: + raise HTTPException( + status_code=400, + detail=f"Invalid order_by field: '{order_by}'. Allowed fields: {', '.join(sorted(allowed_fields))}", + ) + validated_order_by = parts[0] + elif len(parts) == 2: + if parts[0] not in allowed_fields or parts[1] not in allowed_directions: + raise HTTPException( + status_code=400, + detail=f"Invalid order_by: '{order_by}'. Allowed fields: {', '.join(sorted(allowed_fields))}. Allowed directions: asc, desc", + ) + validated_order_by = f"{parts[0]} {parts[1]}" + else: + raise HTTPException( + status_code=400, + detail=f"Invalid order_by format: '{order_by}'. Expected 'field' or 'field direction'", + ) + + # Build the query with counts + query = f""" + SELECT *, + count(<-reference.in) as source_count, + count(<-artifact.in) as note_count + FROM notebook + ORDER BY {validated_order_by} + """ + + result = await repo_query(query) + + # Filter by archived status if specified + if archived is not None: + result = [nb for nb in result if nb.get("archived") == archived] + + return [ + NotebookResponse( + id=str(nb.get("id", "")), + name=nb.get("name", ""), + description=nb.get("description", ""), + archived=nb.get("archived", False), + created=str(nb.get("created", "")), + updated=str(nb.get("updated", "")), + source_count=nb.get("source_count", 0), + note_count=nb.get("note_count", 0), + ) + for nb in result + ] + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching notebooks: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching notebooks: {str(e)}" + ) + + +@router.post("/notebooks", response_model=NotebookResponse) +async def create_notebook(notebook: NotebookCreate): + """Create a new notebook.""" + try: + new_notebook = Notebook( + name=notebook.name, + description=notebook.description, + ) + await new_notebook.save() + + return NotebookResponse( + id=new_notebook.id or "", + name=new_notebook.name, + description=new_notebook.description, + archived=new_notebook.archived or False, + created=str(new_notebook.created), + updated=str(new_notebook.updated), + source_count=0, # New notebook has no sources + note_count=0, # New notebook has no notes + ) + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error creating notebook: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error creating notebook: {str(e)}" + ) + + +@router.get( + "/notebooks/{notebook_id}/delete-preview", response_model=NotebookDeletePreview +) +async def get_notebook_delete_preview(notebook_id: str): + """Get a preview of what will be deleted when this notebook is deleted.""" + try: + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + preview = await notebook.get_delete_preview() + + return NotebookDeletePreview( + notebook_id=str(notebook.id), + notebook_name=notebook.name, + note_count=preview["note_count"], + exclusive_source_count=preview["exclusive_source_count"], + shared_source_count=preview["shared_source_count"], + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting delete preview for notebook {notebook_id}: {e}") + raise HTTPException( + status_code=500, + detail=f"Error fetching notebook deletion preview: {str(e)}", + ) + + +@router.get("/notebooks/{notebook_id}", response_model=NotebookResponse) +async def get_notebook(notebook_id: str): + """Get a specific notebook by ID.""" + try: + # Query with counts for single notebook + query = """ + SELECT *, + count(<-reference.in) as source_count, + count(<-artifact.in) as note_count + FROM $notebook_id + """ + result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)}) + + if not result: + raise HTTPException(status_code=404, detail="Notebook not found") + + nb = result[0] + return NotebookResponse( + id=str(nb.get("id", "")), + name=nb.get("name", ""), + description=nb.get("description", ""), + archived=nb.get("archived", False), + created=str(nb.get("created", "")), + updated=str(nb.get("updated", "")), + source_count=nb.get("source_count", 0), + note_count=nb.get("note_count", 0), + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching notebook {notebook_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching notebook: {str(e)}" + ) + + +@router.put("/notebooks/{notebook_id}", response_model=NotebookResponse) +async def update_notebook(notebook_id: str, notebook_update: NotebookUpdate): + """Update a notebook.""" + try: + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Update only provided fields + if notebook_update.name is not None: + notebook.name = notebook_update.name + if notebook_update.description is not None: + notebook.description = notebook_update.description + if notebook_update.archived is not None: + notebook.archived = notebook_update.archived + + await notebook.save() + + # Query with counts after update + query = """ + SELECT *, + count(<-reference.in) as source_count, + count(<-artifact.in) as note_count + FROM $notebook_id + """ + result = await repo_query(query, {"notebook_id": ensure_record_id(notebook_id)}) + + if result: + nb = result[0] + return NotebookResponse( + id=str(nb.get("id", "")), + name=nb.get("name", ""), + description=nb.get("description", ""), + archived=nb.get("archived", False), + created=str(nb.get("created", "")), + updated=str(nb.get("updated", "")), + source_count=nb.get("source_count", 0), + note_count=nb.get("note_count", 0), + ) + + # Fallback if query fails + return NotebookResponse( + id=notebook.id or "", + name=notebook.name, + description=notebook.description, + archived=notebook.archived or False, + created=str(notebook.created), + updated=str(notebook.updated), + source_count=0, + note_count=0, + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error updating notebook {notebook_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error updating notebook: {str(e)}" + ) + + +@router.post("/notebooks/{notebook_id}/sources/{source_id}") +async def add_source_to_notebook(notebook_id: str, source_id: str): + """Add an existing source to a notebook (create the reference).""" + try: + # Check if notebook exists + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Check if source exists + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Check if reference already exists (idempotency) + existing_ref = await repo_query( + "SELECT * FROM reference WHERE out = $source_id AND in = $notebook_id", + { + "notebook_id": ensure_record_id(notebook_id), + "source_id": ensure_record_id(source_id), + }, + ) + + # If reference doesn't exist, create it + if not existing_ref: + await repo_query( + "RELATE $source_id->reference->$notebook_id", + { + "notebook_id": ensure_record_id(notebook_id), + "source_id": ensure_record_id(source_id), + }, + ) + + return {"message": "Source linked to notebook successfully"} + except HTTPException: + raise + except Exception as e: + logger.error( + f"Error linking source {source_id} to notebook {notebook_id}: {str(e)}" + ) + raise HTTPException( + status_code=500, detail=f"Error linking source to notebook: {str(e)}" + ) + + +@router.delete("/notebooks/{notebook_id}/sources/{source_id}") +async def remove_source_from_notebook(notebook_id: str, source_id: str): + """Remove a source from a notebook (delete the reference).""" + try: + # Check if notebook exists + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Delete the reference record linking source to notebook + await repo_query( + "DELETE FROM reference WHERE out = $notebook_id AND in = $source_id", + { + "notebook_id": ensure_record_id(notebook_id), + "source_id": ensure_record_id(source_id), + }, + ) + + return {"message": "Source removed from notebook successfully"} + except HTTPException: + raise + except Exception as e: + logger.error( + f"Error removing source {source_id} from notebook {notebook_id}: {str(e)}" + ) + raise HTTPException( + status_code=500, detail=f"Error removing source from notebook: {str(e)}" + ) + + +@router.delete("/notebooks/{notebook_id}", response_model=NotebookDeleteResponse) +async def delete_notebook( + notebook_id: str, + delete_exclusive_sources: bool = Query( + False, + description="Whether to delete sources that belong only to this notebook", + ), +): + """ + Delete a notebook with cascade deletion. + + Always deletes all notes associated with the notebook. + If delete_exclusive_sources is True, also deletes sources that belong only + to this notebook (not linked to any other notebooks). + """ + try: + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + result = await notebook.delete(delete_exclusive_sources=delete_exclusive_sources) + + return NotebookDeleteResponse( + message="Notebook deleted successfully", + deleted_notes=result["deleted_notes"], + deleted_sources=result["deleted_sources"], + unlinked_sources=result["unlinked_sources"], + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting notebook {notebook_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error deleting notebook: {str(e)}" + ) diff --git a/api/routers/notes.py b/api/routers/notes.py new file mode 100644 index 0000000000000000000000000000000000000000..ddc8f15d885c3f99ccc9a9f74dd69366144e0558 --- /dev/null +++ b/api/routers/notes.py @@ -0,0 +1,189 @@ +from typing import List, Literal, Optional + +from fastapi import APIRouter, HTTPException, Query +from loguru import logger + +from api.models import NoteCreate, NoteResponse, NoteUpdate +from open_notebook.domain.notebook import Note +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +@router.get("/notes", response_model=List[NoteResponse]) +async def get_notes( + notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"), +): + """Get all notes with optional notebook filtering.""" + try: + if notebook_id: + # Get notes for a specific notebook + from open_notebook.domain.notebook import Notebook + + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + notes = await notebook.get_notes() + else: + # Get all notes + notes = await Note.get_all(order_by="updated desc") + + return [ + NoteResponse( + id=note.id or "", + title=note.title, + content=note.content, + note_type=note.note_type, + created=str(note.created), + updated=str(note.updated), + ) + for note in notes + ] + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching notes: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching notes: {str(e)}") + + +@router.post("/notes", response_model=NoteResponse) +async def create_note(note_data: NoteCreate): + """Create a new note.""" + try: + # Auto-generate title if not provided and it's an AI note + title = note_data.title + if not title and note_data.note_type == "ai" and note_data.content: + from open_notebook.graphs.prompt import graph as prompt_graph + + prompt = "Based on the Note below, please provide a Title for this content, with max 15 words" + result = await prompt_graph.ainvoke( + { # type: ignore[arg-type] + "input_text": note_data.content, + "prompt": prompt, + } + ) + title = result.get("output", "Untitled Note") + + # Validate note_type + note_type: Optional[Literal["human", "ai"]] = None + if note_data.note_type in ("human", "ai"): + note_type = note_data.note_type # type: ignore[assignment] + elif note_data.note_type is not None: + raise HTTPException( + status_code=400, detail="note_type must be 'human' or 'ai'" + ) + + new_note = Note( + title=title, + content=note_data.content, + note_type=note_type, + ) + command_id = await new_note.save() + + # Add to notebook if specified + if note_data.notebook_id: + from open_notebook.domain.notebook import Notebook + + notebook = await Notebook.get(note_data.notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + await new_note.add_to_notebook(note_data.notebook_id) + + return NoteResponse( + id=new_note.id or "", + title=new_note.title, + content=new_note.content, + note_type=new_note.note_type, + created=str(new_note.created), + updated=str(new_note.updated), + command_id=str(command_id) if command_id else None, + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error creating note: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error creating note: {str(e)}") + + +@router.get("/notes/{note_id}", response_model=NoteResponse) +async def get_note(note_id: str): + """Get a specific note by ID.""" + try: + note = await Note.get(note_id) + if not note: + raise HTTPException(status_code=404, detail="Note not found") + + return NoteResponse( + id=note.id or "", + title=note.title, + content=note.content, + note_type=note.note_type, + created=str(note.created), + updated=str(note.updated), + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching note {note_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching note: {str(e)}") + + +@router.put("/notes/{note_id}", response_model=NoteResponse) +async def update_note(note_id: str, note_update: NoteUpdate): + """Update a note.""" + try: + note = await Note.get(note_id) + if not note: + raise HTTPException(status_code=404, detail="Note not found") + + # Update only provided fields + if note_update.title is not None: + note.title = note_update.title + if note_update.content is not None: + note.content = note_update.content + if note_update.note_type is not None: + if note_update.note_type in ("human", "ai"): + note.note_type = note_update.note_type # type: ignore[assignment] + else: + raise HTTPException( + status_code=400, detail="note_type must be 'human' or 'ai'" + ) + + command_id = await note.save() + + return NoteResponse( + id=note.id or "", + title=note.title, + content=note.content, + note_type=note.note_type, + created=str(note.created), + updated=str(note.updated), + command_id=str(command_id) if command_id else None, + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error updating note {note_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error updating note: {str(e)}") + + +@router.delete("/notes/{note_id}") +async def delete_note(note_id: str): + """Delete a note.""" + try: + note = await Note.get(note_id) + if not note: + raise HTTPException(status_code=404, detail="Note not found") + + await note.delete() + + return {"message": "Note deleted successfully"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting note {note_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error deleting note: {str(e)}") diff --git a/api/routers/podcasts.py b/api/routers/podcasts.py new file mode 100644 index 0000000000000000000000000000000000000000..042f66a873094a1f5dd7cbf272442ea1be62c9e7 --- /dev/null +++ b/api/routers/podcasts.py @@ -0,0 +1,299 @@ +from pathlib import Path +from typing import List, Optional +from urllib.parse import unquote, urlparse + +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse +from loguru import logger +from pydantic import BaseModel + +from api.podcast_service import ( + PodcastGenerationRequest, + PodcastGenerationResponse, + PodcastService, +) + +router = APIRouter() + + +class PodcastEpisodeResponse(BaseModel): + id: str + name: str + episode_profile: dict + speaker_profile: dict + briefing: str + audio_file: Optional[str] = None + audio_url: Optional[str] = None + transcript: Optional[dict] = None + outline: Optional[dict] = None + created: Optional[str] = None + job_status: Optional[str] = None + error_message: Optional[str] = None + + +def _resolve_audio_path(audio_file: str) -> Path: + if audio_file.startswith("file://"): + parsed = urlparse(audio_file) + return Path(unquote(parsed.path)) + return Path(audio_file) + + +@router.post("/podcasts/generate", response_model=PodcastGenerationResponse) +async def generate_podcast(request: PodcastGenerationRequest): + """ + Generate a podcast episode using Episode Profiles. + Returns immediately with job ID for status tracking. + """ + try: + job_id = await PodcastService.submit_generation_job( + episode_profile_name=request.episode_profile, + speaker_profile_name=request.speaker_profile, + episode_name=request.episode_name, + notebook_id=request.notebook_id, + content=request.content, + briefing_suffix=request.briefing_suffix, + ) + + return PodcastGenerationResponse( + job_id=job_id, + status="submitted", + message=f"Podcast generation started for episode '{request.episode_name}'", + episode_profile=request.episode_profile, + episode_name=request.episode_name, + ) + + except Exception as e: + logger.error(f"Error generating podcast: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to generate podcast" + ) + + +@router.get("/podcasts/jobs/{job_id}") +async def get_podcast_job_status(job_id: str): + """Get the status of a podcast generation job""" + try: + status_data = await PodcastService.get_job_status(job_id) + return status_data + + except Exception as e: + logger.error(f"Error fetching podcast job status: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to fetch job status" + ) + + +@router.get("/podcasts/episodes", response_model=List[PodcastEpisodeResponse]) +async def list_podcast_episodes(): + """List all podcast episodes""" + try: + episodes = await PodcastService.list_episodes() + + response_episodes = [] + for episode in episodes: + # Skip incomplete episodes without command or audio + if not episode.command and not episode.audio_file: + continue + + # Get job status and error message if available + job_status = None + error_message = None + if episode.command: + try: + detail = await episode.get_job_detail() + job_status = detail["status"] + error_message = detail["error_message"] + except Exception: + job_status = "unknown" + else: + # No command but has audio file = completed import + job_status = "completed" + + audio_url = None + if episode.audio_file: + audio_path = _resolve_audio_path(episode.audio_file) + if audio_path.exists(): + audio_url = f"/api/podcasts/episodes/{episode.id}/audio" + + response_episodes.append( + PodcastEpisodeResponse( + id=str(episode.id), + name=episode.name, + episode_profile=episode.episode_profile, + speaker_profile=episode.speaker_profile, + briefing=episode.briefing, + audio_file=episode.audio_file, + audio_url=audio_url, + transcript=episode.transcript, + outline=episode.outline, + created=str(episode.created) if episode.created else None, + job_status=job_status, + error_message=error_message, + ) + ) + + return response_episodes + + except Exception as e: + logger.error(f"Error listing podcast episodes: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to list podcast episodes" + ) + + +@router.get("/podcasts/episodes/{episode_id}", response_model=PodcastEpisodeResponse) +async def get_podcast_episode(episode_id: str): + """Get a specific podcast episode""" + try: + episode = await PodcastService.get_episode(episode_id) + + # Get job status and error message if available + job_status = None + error_message = None + if episode.command: + try: + detail = await episode.get_job_detail() + job_status = detail["status"] + error_message = detail["error_message"] + except Exception: + job_status = "unknown" + else: + # No command but has audio file = completed import + job_status = "completed" if episode.audio_file else "unknown" + + audio_url = None + if episode.audio_file: + audio_path = _resolve_audio_path(episode.audio_file) + if audio_path.exists(): + audio_url = f"/api/podcasts/episodes/{episode.id}/audio" + + return PodcastEpisodeResponse( + id=str(episode.id), + name=episode.name, + episode_profile=episode.episode_profile, + speaker_profile=episode.speaker_profile, + briefing=episode.briefing, + audio_file=episode.audio_file, + audio_url=audio_url, + transcript=episode.transcript, + outline=episode.outline, + created=str(episode.created) if episode.created else None, + job_status=job_status, + error_message=error_message, + ) + + except Exception as e: + logger.error(f"Error fetching podcast episode: {str(e)}") + raise HTTPException(status_code=404, detail="Episode not found") + + +@router.get("/podcasts/episodes/{episode_id}/audio") +async def stream_podcast_episode_audio(episode_id: str): + """Stream the audio file associated with a podcast episode""" + try: + episode = await PodcastService.get_episode(episode_id) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching podcast episode for audio: {str(e)}") + raise HTTPException(status_code=404, detail="Episode not found") + + if not episode.audio_file: + raise HTTPException(status_code=404, detail="Episode has no audio file") + + audio_path = _resolve_audio_path(episode.audio_file) + if not audio_path.exists(): + raise HTTPException(status_code=404, detail="Audio file not found on disk") + + return FileResponse( + audio_path, + media_type="audio/mpeg", + filename=audio_path.name, + ) + + +@router.post("/podcasts/episodes/{episode_id}/retry") +async def retry_podcast_episode(episode_id: str): + """Retry a failed podcast episode by deleting it and submitting a new job""" + try: + episode = await PodcastService.get_episode(episode_id) + + # Validate episode is in a failed state + detail = await episode.get_job_detail() + if detail["status"] not in ("failed", "error"): + raise HTTPException( + status_code=400, + detail=f"Episode is not in a failed state (current: {detail['status']})", + ) + + # Extract params for re-submission + ep_profile_name = episode.episode_profile.get("name") + sp_profile_name = episode.speaker_profile.get("name") + episode_name = episode.name + content = episode.content + + if not ep_profile_name or not sp_profile_name: + raise HTTPException( + status_code=400, + detail="Cannot retry: episode or speaker profile name missing from stored data", + ) + + # Delete audio file if any + if episode.audio_file: + audio_path = _resolve_audio_path(episode.audio_file) + if audio_path.exists(): + try: + audio_path.unlink() + except Exception as e: + logger.warning(f"Failed to delete audio file {audio_path}: {e}") + + # Delete the failed episode + await episode.delete() + + # Submit a new job + job_id = await PodcastService.submit_generation_job( + episode_profile_name=ep_profile_name, + speaker_profile_name=sp_profile_name, + episode_name=episode_name, + content=content, + ) + + return {"job_id": job_id, "message": "Retry submitted successfully"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error retrying podcast episode: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to retry episode" + ) + + +@router.delete("/podcasts/episodes/{episode_id}") +async def delete_podcast_episode(episode_id: str): + """Delete a podcast episode and its associated audio file""" + try: + # Get the episode first to check if it exists and get the audio file path + episode = await PodcastService.get_episode(episode_id) + + # Delete the physical audio file if it exists + if episode.audio_file: + audio_path = _resolve_audio_path(episode.audio_file) + if audio_path.exists(): + try: + audio_path.unlink() + logger.info(f"Deleted audio file: {audio_path}") + except Exception as e: + logger.warning(f"Failed to delete audio file {audio_path}: {e}") + + # Delete the episode from the database + await episode.delete() + + logger.info(f"Deleted podcast episode: {episode_id}") + return {"message": "Episode deleted successfully", "episode_id": episode_id} + + except Exception as e: + logger.error(f"Error deleting podcast episode: {str(e)}") + raise HTTPException( + status_code=500, detail="Failed to delete episode" + ) diff --git a/api/routers/search.py b/api/routers/search.py new file mode 100644 index 0000000000000000000000000000000000000000..53c7d768336a4202d0cb6a347d4e50bc38e9d298 --- /dev/null +++ b/api/routers/search.py @@ -0,0 +1,217 @@ +import json +from typing import AsyncGenerator + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse +from loguru import logger + +from api.models import AskRequest, AskResponse, SearchRequest, SearchResponse +from open_notebook.ai.models import Model, model_manager +from open_notebook.domain.notebook import text_search, vector_search +from open_notebook.exceptions import DatabaseOperationError, InvalidInputError +from open_notebook.graphs.ask import graph as ask_graph + +router = APIRouter() + + +@router.post("/search", response_model=SearchResponse) +async def search_knowledge_base(search_request: SearchRequest): + """Search the knowledge base using text or vector search.""" + try: + if search_request.type == "vector": + # Check if embedding model is available for vector search + if not await model_manager.get_embedding_model(): + raise HTTPException( + status_code=400, + detail="Vector search requires an embedding model. Please configure one in the Models section.", + ) + + results = await vector_search( + keyword=search_request.query, + results=search_request.limit, + source=search_request.search_sources, + note=search_request.search_notes, + minimum_score=search_request.minimum_score, + ) + else: + # Text search + results = await text_search( + keyword=search_request.query, + results=search_request.limit, + source=search_request.search_sources, + note=search_request.search_notes, + ) + + return SearchResponse( + results=results or [], + total_count=len(results) if results else 0, + search_type=search_request.type, + ) + + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except DatabaseOperationError as e: + logger.error(f"Database error during search: {str(e)}") + raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error during search: {str(e)}") + raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") + + +async def stream_ask_response( + question: str, strategy_model: Model, answer_model: Model, final_answer_model: Model +) -> AsyncGenerator[str, None]: + """Stream the ask response as Server-Sent Events.""" + try: + final_answer = None + + async for chunk in ask_graph.astream( + input=dict(question=question), # type: ignore[arg-type] + config=dict( + configurable=dict( + strategy_model=strategy_model.id, + answer_model=answer_model.id, + final_answer_model=final_answer_model.id, + ) + ), + stream_mode="updates", + ): + if "agent" in chunk: + strategy_data = { + "type": "strategy", + "reasoning": chunk["agent"]["strategy"].reasoning, + "searches": [ + {"term": search.term, "instructions": search.instructions} + for search in chunk["agent"]["strategy"].searches + ], + } + yield f"data: {json.dumps(strategy_data)}\n\n" + + elif "provide_answer" in chunk: + for answer in chunk["provide_answer"]["answers"]: + answer_data = {"type": "answer", "content": answer} + yield f"data: {json.dumps(answer_data)}\n\n" + + elif "write_final_answer" in chunk: + final_answer = chunk["write_final_answer"]["final_answer"] + final_data = {"type": "final_answer", "content": final_answer} + yield f"data: {json.dumps(final_data)}\n\n" + + # Send completion signal + completion_data = {"type": "complete", "final_answer": final_answer} + yield f"data: {json.dumps(completion_data)}\n\n" + + except Exception as e: + from open_notebook.utils.error_classifier import classify_error + + _, user_message = classify_error(e) + logger.error(f"Error in ask streaming: {str(e)}") + error_data = {"type": "error", "message": user_message} + yield f"data: {json.dumps(error_data)}\n\n" + + +@router.post("/search/ask") +async def ask_knowledge_base(ask_request: AskRequest): + """Ask the knowledge base a question using AI models.""" + try: + # Validate models exist + strategy_model = await Model.get(ask_request.strategy_model) + answer_model = await Model.get(ask_request.answer_model) + final_answer_model = await Model.get(ask_request.final_answer_model) + + if not strategy_model: + raise HTTPException( + status_code=400, + detail=f"Strategy model {ask_request.strategy_model} not found", + ) + if not answer_model: + raise HTTPException( + status_code=400, + detail=f"Answer model {ask_request.answer_model} not found", + ) + if not final_answer_model: + raise HTTPException( + status_code=400, + detail=f"Final answer model {ask_request.final_answer_model} not found", + ) + + # Check if embedding model is available + if not await model_manager.get_embedding_model(): + raise HTTPException( + status_code=400, + detail="Ask feature requires an embedding model. Please configure one in the Models section.", + ) + + # For streaming response + return StreamingResponse( + stream_ask_response( + ask_request.question, strategy_model, answer_model, final_answer_model + ), + media_type="text/plain", + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in ask endpoint: {str(e)}") + raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}") + + +@router.post("/search/ask/simple", response_model=AskResponse) +async def ask_knowledge_base_simple(ask_request: AskRequest): + """Ask the knowledge base a question and return a simple response (non-streaming).""" + try: + # Validate models exist + strategy_model = await Model.get(ask_request.strategy_model) + answer_model = await Model.get(ask_request.answer_model) + final_answer_model = await Model.get(ask_request.final_answer_model) + + if not strategy_model: + raise HTTPException( + status_code=400, + detail=f"Strategy model {ask_request.strategy_model} not found", + ) + if not answer_model: + raise HTTPException( + status_code=400, + detail=f"Answer model {ask_request.answer_model} not found", + ) + if not final_answer_model: + raise HTTPException( + status_code=400, + detail=f"Final answer model {ask_request.final_answer_model} not found", + ) + + # Check if embedding model is available + if not await model_manager.get_embedding_model(): + raise HTTPException( + status_code=400, + detail="Ask feature requires an embedding model. Please configure one in the Models section.", + ) + + # Run the ask graph and get final result + final_answer = None + async for chunk in ask_graph.astream( + input=dict(question=ask_request.question), # type: ignore[arg-type] + config=dict( + configurable=dict( + strategy_model=strategy_model.id, + answer_model=answer_model.id, + final_answer_model=final_answer_model.id, + ) + ), + stream_mode="updates", + ): + if "write_final_answer" in chunk: + final_answer = chunk["write_final_answer"]["final_answer"] + + if not final_answer: + raise HTTPException(status_code=500, detail="No answer generated") + + return AskResponse(answer=final_answer, question=ask_request.question) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in ask simple endpoint: {str(e)}") + raise HTTPException(status_code=500, detail=f"Ask operation failed: {str(e)}") diff --git a/api/routers/settings.py b/api/routers/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..5e99261920002f119a48ecf7b911315ddcfcfac7 --- /dev/null +++ b/api/routers/settings.py @@ -0,0 +1,88 @@ +from fastapi import APIRouter, HTTPException +from loguru import logger + +from api.models import SettingsResponse, SettingsUpdate +from open_notebook.domain.content_settings import ContentSettings +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +@router.get("/settings", response_model=SettingsResponse) +async def get_settings(): + """Get all application settings.""" + try: + settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment] + + return SettingsResponse( + default_content_processing_engine_doc=settings.default_content_processing_engine_doc, + default_content_processing_engine_url=settings.default_content_processing_engine_url, + default_embedding_option=settings.default_embedding_option, + auto_delete_files=settings.auto_delete_files, + youtube_preferred_languages=settings.youtube_preferred_languages, + ) + except Exception as e: + logger.error(f"Error fetching settings: {str(e)}") + raise HTTPException( + status_code=500, detail="Error fetching settings" + ) + + +@router.put("/settings", response_model=SettingsResponse) +async def update_settings(settings_update: SettingsUpdate): + """Update application settings.""" + try: + settings: ContentSettings = await ContentSettings.get_instance() # type: ignore[assignment] + + # Update only provided fields + if settings_update.default_content_processing_engine_doc is not None: + # Cast to proper literal type + from typing import Literal, cast + + settings.default_content_processing_engine_doc = cast( + Literal["auto", "docling", "simple"], + settings_update.default_content_processing_engine_doc, + ) + if settings_update.default_content_processing_engine_url is not None: + from typing import Literal, cast + + settings.default_content_processing_engine_url = cast( + Literal["auto", "firecrawl", "jina", "simple"], + settings_update.default_content_processing_engine_url, + ) + if settings_update.default_embedding_option is not None: + from typing import Literal, cast + + settings.default_embedding_option = cast( + Literal["ask", "always", "never"], + settings_update.default_embedding_option, + ) + if settings_update.auto_delete_files is not None: + from typing import Literal, cast + + settings.auto_delete_files = cast( + Literal["yes", "no"], settings_update.auto_delete_files + ) + if settings_update.youtube_preferred_languages is not None: + settings.youtube_preferred_languages = ( + settings_update.youtube_preferred_languages + ) + + await settings.update() + + return SettingsResponse( + default_content_processing_engine_doc=settings.default_content_processing_engine_doc, + default_content_processing_engine_url=settings.default_content_processing_engine_url, + default_embedding_option=settings.default_embedding_option, + auto_delete_files=settings.auto_delete_files, + youtube_preferred_languages=settings.youtube_preferred_languages, + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error updating settings: {str(e)}") + raise HTTPException( + status_code=500, detail="Error updating settings" + ) diff --git a/api/routers/source_chat.py b/api/routers/source_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..ca304de04e06106c876269545bb3077276d3aefe --- /dev/null +++ b/api/routers/source_chat.py @@ -0,0 +1,554 @@ +import asyncio +import json +from typing import AsyncGenerator, List, Optional + +from fastapi import APIRouter, HTTPException, Path +from fastapi.responses import StreamingResponse +from langchain_core.messages import HumanMessage +from langchain_core.runnables import RunnableConfig +from loguru import logger +from pydantic import BaseModel, Field + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.notebook import ChatSession, Source +from open_notebook.exceptions import ( + NotFoundError, +) +from open_notebook.graphs.source_chat import source_chat_graph as source_chat_graph +from open_notebook.utils.graph_utils import get_session_message_count + +router = APIRouter() + + +# Request/Response models +class CreateSourceChatSessionRequest(BaseModel): + source_id: str = Field(..., description="Source ID to create chat session for") + title: Optional[str] = Field(None, description="Optional session title") + model_override: Optional[str] = Field( + None, description="Optional model override for this session" + ) + +class UpdateSourceChatSessionRequest(BaseModel): + title: Optional[str] = Field(None, description="New session title") + model_override: Optional[str] = Field( + None, description="Model override for this session" + ) + +class ChatMessage(BaseModel): + id: str = Field(..., description="Message ID") + type: str = Field(..., description="Message type (human|ai)") + content: str = Field(..., description="Message content") + timestamp: Optional[str] = Field(None, description="Message timestamp") + + +class ContextIndicator(BaseModel): + sources: List[str] = Field( + default_factory=list, description="Source IDs used in context" + ) + insights: List[str] = Field( + default_factory=list, description="Insight IDs used in context" + ) + notes: List[str] = Field( + default_factory=list, description="Note IDs used in context" + ) + +class SourceChatSessionResponse(BaseModel): + id: str = Field(..., description="Session ID") + title: str = Field(..., description="Session title") + source_id: str = Field(..., description="Source ID") + model_override: Optional[str] = Field( + None, description="Model override for this session" + ) + created: str = Field(..., description="Creation timestamp") + updated: str = Field(..., description="Last update timestamp") + message_count: Optional[int] = Field( + None, description="Number of messages in session" + ) + +class SourceChatSessionWithMessagesResponse(SourceChatSessionResponse): + messages: List[ChatMessage] = Field( + default_factory=list, description="Session messages" + ) + context_indicators: Optional[ContextIndicator] = Field( + None, description="Context indicators from last response" + ) + +class SendMessageRequest(BaseModel): + message: str = Field(..., description="User message content") + model_override: Optional[str] = Field( + None, description="Optional model override for this message" + ) + +class SuccessResponse(BaseModel): + success: bool = Field(True, description="Operation success status") + message: str = Field(..., description="Success message") + + +@router.post( + "/sources/{source_id}/chat/sessions", response_model=SourceChatSessionResponse +) +async def create_source_chat_session( + request: CreateSourceChatSessionRequest, + source_id: str = Path(..., description="Source ID"), +): + """Create a new chat session for a source.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Create new session with model_override support + session = ChatSession( + title=request.title or f"Source Chat {asyncio.get_event_loop().time():.0f}", + model_override=request.model_override, + ) + await session.save() + + # Relate session to source using "refers_to" relation + await session.relate("refers_to", full_source_id) + + return SourceChatSessionResponse( + id=session.id or "", + title=session.title or "Untitled Session", + source_id=source_id, + model_override=session.model_override, + created=str(session.created), + updated=str(session.updated), + message_count=0, + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Source not found") + except Exception as e: + logger.error(f"Error creating source chat session: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error creating source chat session: {str(e)}" + ) + + +@router.get( + "/sources/{source_id}/chat/sessions", response_model=List[SourceChatSessionResponse] +) +async def get_source_chat_sessions(source_id: str = Path(..., description="Source ID")): + """Get all chat sessions for a source.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Get sessions that refer to this source - first get relations, then sessions + relations = await repo_query( + "SELECT in FROM refers_to WHERE out = $source_id", + {"source_id": ensure_record_id(full_source_id)}, + ) + + sessions = [] + for relation in relations: + session_id_raw = relation.get("in") + if session_id_raw: + session_id = str(session_id_raw) + + session_result = await repo_query( + "SELECT * FROM $id", {"id": ensure_record_id(session_id)} + ) + if session_result and len(session_result) > 0: + session_data = session_result[0] + + # Get message count from LangGraph state + msg_count = await get_session_message_count( + source_chat_graph, session_id + ) + + sessions.append( + SourceChatSessionResponse( + id=session_data.get("id") or "", + title=session_data.get("title") or "Untitled Session", + source_id=source_id, + model_override=session_data.get("model_override"), + created=str(session_data.get("created")), + updated=str(session_data.get("updated")), + message_count=msg_count, + ) + ) + + # Sort sessions by created date (newest first) + sessions.sort(key=lambda x: x.created, reverse=True) + return sessions + except NotFoundError: + raise HTTPException(status_code=404, detail="Source not found") + except Exception as e: + logger.error(f"Error fetching source chat sessions: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching source chat sessions: {str(e)}" + ) + + +@router.get( + "/sources/{source_id}/chat/sessions/{session_id}", + response_model=SourceChatSessionWithMessagesResponse, +) +async def get_source_chat_session( + source_id: str = Path(..., description="Source ID"), + session_id: str = Path(..., description="Session ID"), +): + """Get a specific source chat session with its messages.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Get session + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Verify session is related to this source + relation_query = await repo_query( + "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", + { + "session_id": ensure_record_id(full_session_id), + "source_id": ensure_record_id(full_source_id), + }, + ) + + if not relation_query: + raise HTTPException( + status_code=404, detail="Session not found for this source" + ) + + # Get session state from LangGraph to retrieve messages + # Use sync get_state() in a thread since SqliteSaver doesn't support async + thread_state = await asyncio.to_thread( + source_chat_graph.get_state, + config=RunnableConfig(configurable={"thread_id": full_session_id}), + ) + + # Extract messages from state + messages: list[ChatMessage] = [] + context_indicators = None + + if thread_state and thread_state.values: + # Extract messages + if "messages" in thread_state.values: + for msg in thread_state.values["messages"]: + messages.append( + ChatMessage( + id=getattr(msg, "id", f"msg_{len(messages)}"), + type=msg.type if hasattr(msg, "type") else "unknown", + content=msg.content + if hasattr(msg, "content") + else str(msg), + timestamp=None, # LangChain messages don't have timestamps by default + ) + ) + + # Extract context indicators from the last state + if "context_indicators" in thread_state.values: + context_data = thread_state.values["context_indicators"] + context_indicators = ContextIndicator( + sources=context_data.get("sources", []), + insights=context_data.get("insights", []), + notes=context_data.get("notes", []), + ) + + return SourceChatSessionWithMessagesResponse( + id=session.id or "", + title=session.title or "Untitled Session", + source_id=source_id, + model_override=getattr(session, "model_override", None), + created=str(session.created), + updated=str(session.updated), + message_count=len(messages), + messages=messages, + context_indicators=context_indicators, + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Source or session not found") + except Exception as e: + logger.error(f"Error fetching source chat session: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching source chat session: {str(e)}" + ) + + +@router.put( + "/sources/{source_id}/chat/sessions/{session_id}", + response_model=SourceChatSessionResponse, +) +async def update_source_chat_session( + request: UpdateSourceChatSessionRequest, + source_id: str = Path(..., description="Source ID"), + session_id: str = Path(..., description="Session ID"), +): + """Update source chat session title and/or model override.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Get session + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Verify session is related to this source + relation_query = await repo_query( + "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", + { + "session_id": ensure_record_id(full_session_id), + "source_id": ensure_record_id(full_source_id), + }, + ) + + if not relation_query: + raise HTTPException( + status_code=404, detail="Session not found for this source" + ) + + # Update session fields + if request.title is not None: + session.title = request.title + if request.model_override is not None: + session.model_override = request.model_override + + await session.save() + + # Get message count from LangGraph state + msg_count = await get_session_message_count(source_chat_graph, full_session_id) + + return SourceChatSessionResponse( + id=session.id or "", + title=session.title or "Untitled Session", + source_id=source_id, + model_override=getattr(session, "model_override", None), + created=str(session.created), + updated=str(session.updated), + message_count=msg_count, + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Source or session not found") + except Exception as e: + logger.error(f"Error updating source chat session: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error updating source chat session: {str(e)}" + ) + + +@router.delete( + "/sources/{source_id}/chat/sessions/{session_id}", response_model=SuccessResponse +) +async def delete_source_chat_session( + source_id: str = Path(..., description="Source ID"), + session_id: str = Path(..., description="Session ID"), +): + """Delete a source chat session.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Get session + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Verify session is related to this source + relation_query = await repo_query( + "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", + { + "session_id": ensure_record_id(full_session_id), + "source_id": ensure_record_id(full_source_id), + }, + ) + + if not relation_query: + raise HTTPException( + status_code=404, detail="Session not found for this source" + ) + + await session.delete() + + return SuccessResponse( + success=True, message="Source chat session deleted successfully" + ) + except NotFoundError: + raise HTTPException(status_code=404, detail="Source or session not found") + except Exception as e: + logger.error(f"Error deleting source chat session: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error deleting source chat session: {str(e)}" + ) + + +async def stream_source_chat_response( + session_id: str, source_id: str, message: str, model_override: Optional[str] = None +) -> AsyncGenerator[str, None]: + """Stream the source chat response as Server-Sent Events.""" + try: + # Get current state + # Use sync get_state() in a thread since SqliteSaver doesn't support async + current_state = await asyncio.to_thread( + source_chat_graph.get_state, + config=RunnableConfig(configurable={"thread_id": session_id}), + ) + + # Prepare state for execution + state_values = current_state.values if current_state else {} + state_values["messages"] = state_values.get("messages", []) + state_values["source_id"] = source_id + state_values["model_override"] = model_override + + # Add user message to state + user_message = HumanMessage(content=message) + state_values["messages"].append(user_message) + + # Send user message event + user_event = {"type": "user_message", "content": message, "timestamp": None} + yield f"data: {json.dumps(user_event)}\n\n" + + # Execute source chat graph synchronously (like notebook chat does) + result = source_chat_graph.invoke( + input=state_values, # type: ignore[arg-type] + config=RunnableConfig( + configurable={"thread_id": session_id, "model_id": model_override} + ), + ) + + # Stream the complete AI response + if "messages" in result: + for msg in result["messages"]: + if hasattr(msg, "type") and msg.type == "ai": + ai_event = { + "type": "ai_message", + "content": msg.content if hasattr(msg, "content") else str(msg), + "timestamp": None, + } + yield f"data: {json.dumps(ai_event)}\n\n" + + # Stream context indicators + if "context_indicators" in result: + context_event = { + "type": "context_indicators", + "data": result["context_indicators"], + } + yield f"data: {json.dumps(context_event)}\n\n" + + # Send completion signal + completion_event = {"type": "complete"} + yield f"data: {json.dumps(completion_event)}\n\n" + + except Exception as e: + from open_notebook.utils.error_classifier import classify_error + + _, user_message = classify_error(e) + logger.error(f"Error in source chat streaming: {str(e)}") + error_event = {"type": "error", "message": user_message} + yield f"data: {json.dumps(error_event)}\n\n" + + +@router.post("/sources/{source_id}/chat/sessions/{session_id}/messages") +async def send_message_to_source_chat( + request: SendMessageRequest, + source_id: str = Path(..., description="Source ID"), + session_id: str = Path(..., description="Session ID"), +): + """Send a message to source chat session with SSE streaming response.""" + try: + # Verify source exists + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + source = await Source.get(full_source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Verify session exists and is related to source + full_session_id = ( + session_id + if session_id.startswith("chat_session:") + else f"chat_session:{session_id}" + ) + session = await ChatSession.get(full_session_id) + if not session: + raise HTTPException(status_code=404, detail="Session not found") + + # Verify session is related to this source + relation_query = await repo_query( + "SELECT * FROM refers_to WHERE in = $session_id AND out = $source_id", + { + "session_id": ensure_record_id(full_session_id), + "source_id": ensure_record_id(full_source_id), + }, + ) + + if not relation_query: + raise HTTPException( + status_code=404, detail="Session not found for this source" + ) + + if not request.message: + raise HTTPException(status_code=400, detail="Message content is required") + + # Determine model override (request override takes precedence over session override) + model_override = request.model_override or getattr( + session, "model_override", None + ) + + # Update session timestamp + await session.save() + + # Return streaming response + return StreamingResponse( + stream_source_chat_response( + session_id=full_session_id, + source_id=full_source_id, + message=request.message, + model_override=model_override, + ), + media_type="text/plain", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Content-Type": "text/plain; charset=utf-8", + }, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error sending message to source chat: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error sending message: {str(e)}") diff --git a/api/routers/sources.py b/api/routers/sources.py new file mode 100644 index 0000000000000000000000000000000000000000..9c4b745238d2bc8f9fd3cb34678a1ffac5f2c38f --- /dev/null +++ b/api/routers/sources.py @@ -0,0 +1,1045 @@ +import asyncio +import os +from pathlib import Path +from typing import Any, List, Optional + +from fastapi import ( + APIRouter, + Depends, + File, + Form, + HTTPException, + Query, + UploadFile, +) +from fastapi.responses import FileResponse, Response +from loguru import logger +from surreal_commands import execute_command_sync, submit_command + +from api.command_service import CommandService +from api.models import ( + AssetModel, + CreateSourceInsightRequest, + InsightCreationResponse, + SourceCreate, + SourceInsightResponse, + SourceListResponse, + SourceResponse, + SourceStatusResponse, + SourceUpdate, +) +from commands.source_commands import SourceProcessingInput +from open_notebook.config import UPLOADS_FOLDER +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.notebook import Asset, Notebook, Source +from open_notebook.domain.transformation import Transformation +from open_notebook.exceptions import InvalidInputError + +router = APIRouter() + + +def generate_unique_filename(original_filename: str, upload_folder: str) -> str: + """Generate unique filename like Streamlit app (append counter if file exists).""" + file_path = Path(upload_folder) + file_path.mkdir(parents=True, exist_ok=True) + + # Strip directory components to prevent path traversal + safe_filename = os.path.basename(original_filename) + if not safe_filename: + raise ValueError("Invalid filename") + + # Split filename and extension + stem = Path(safe_filename).stem + suffix = Path(safe_filename).suffix + + # Check if file exists and generate unique name + counter = 0 + while True: + if counter == 0: + new_filename = safe_filename + else: + new_filename = f"{stem} ({counter}){suffix}" + + full_path = file_path / new_filename + # Verify resolved path stays within upload folder + resolved = full_path.resolve() + if not str(resolved).startswith(str(file_path.resolve()) + os.sep): + raise ValueError("Invalid filename: path traversal detected") + if not resolved.exists(): + return str(resolved) + counter += 1 + + +async def save_uploaded_file(upload_file: UploadFile) -> str: + """Save uploaded file to uploads folder and return file path.""" + if not upload_file.filename: + raise ValueError("No filename provided") + + # Generate unique filename + file_path = generate_unique_filename(upload_file.filename, UPLOADS_FOLDER) + + try: + # Save file + with open(file_path, "wb") as f: + content = await upload_file.read() + f.write(content) + + logger.info(f"Saved uploaded file to: {file_path}") + return file_path + except Exception as e: + logger.error(f"Failed to save uploaded file: {e}") + # Clean up partial file if it exists + if os.path.exists(file_path): + os.unlink(file_path) + raise + + +def parse_source_form_data( + type: str = Form(...), + notebook_id: Optional[str] = Form(None), + notebooks: Optional[str] = Form(None), # JSON string of notebook IDs + url: Optional[str] = Form(None), + content: Optional[str] = Form(None), + title: Optional[str] = Form(None), + transformations: Optional[str] = Form(None), # JSON string of transformation IDs + embed: str = Form("false"), # Accept as string, convert to bool + delete_source: str = Form("false"), # Accept as string, convert to bool + async_processing: str = Form("false"), # Accept as string, convert to bool + file: Optional[UploadFile] = File(None), +) -> tuple[SourceCreate, Optional[UploadFile]]: + """Parse form data into SourceCreate model and return upload file separately.""" + import json + + # Convert string booleans to actual booleans + def str_to_bool(value: str) -> bool: + return value.lower() in ("true", "1", "yes", "on") + + embed_bool = str_to_bool(embed) + delete_source_bool = str_to_bool(delete_source) + async_processing_bool = str_to_bool(async_processing) + + # Parse JSON strings + notebooks_list = None + if notebooks: + try: + notebooks_list = json.loads(notebooks) + except json.JSONDecodeError: + logger.error(f"Invalid JSON in notebooks field: {notebooks}") + raise ValueError("Invalid JSON in notebooks field") + + transformations_list = [] + if transformations: + try: + transformations_list = json.loads(transformations) + except json.JSONDecodeError: + logger.error(f"Invalid JSON in transformations field: {transformations}") + raise ValueError("Invalid JSON in transformations field") + + # Create SourceCreate instance + try: + source_data = SourceCreate( + type=type, + notebook_id=notebook_id, + notebooks=notebooks_list, + url=url, + content=content, + title=title, + file_path=None, # Will be set later if file is uploaded + transformations=transformations_list, + embed=embed_bool, + delete_source=delete_source_bool, + async_processing=async_processing_bool, + ) + pass # SourceCreate instance created successfully + except Exception as e: + logger.error(f"Failed to create SourceCreate instance: {e}") + raise + + return source_data, file + + +@router.get("/sources", response_model=List[SourceListResponse]) +async def get_sources( + notebook_id: Optional[str] = Query(None, description="Filter by notebook ID"), + limit: int = Query( + 50, ge=1, le=100, description="Number of sources to return (1-100)" + ), + offset: int = Query(0, ge=0, description="Number of sources to skip"), + sort_by: str = Query( + "updated", description="Field to sort by (created or updated)" + ), + sort_order: str = Query("desc", description="Sort order (asc or desc)"), +): + """Get sources with pagination and sorting support.""" + try: + # Validate sort parameters + if sort_by not in ["created", "updated"]: + raise HTTPException( + status_code=400, detail="sort_by must be 'created' or 'updated'" + ) + if sort_order.lower() not in ["asc", "desc"]: + raise HTTPException( + status_code=400, detail="sort_order must be 'asc' or 'desc'" + ) + + # Build ORDER BY clause + order_clause = f"ORDER BY {sort_by} {sort_order.upper()}" + + # Build the query + if notebook_id: + # Verify notebook exists first + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException(status_code=404, detail="Notebook not found") + + # Query sources for specific notebook - include command field with FETCH + query = f""" + SELECT id, asset, created, title, updated, topics, command, + (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count, + (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded + FROM (select value in from reference where out=$notebook_id) + {order_clause} + LIMIT $limit START $offset + FETCH command + """ + result = await repo_query( + query, + { + "notebook_id": ensure_record_id(notebook_id), + "limit": limit, + "offset": offset, + }, + ) + else: + # Query all sources - include command field with FETCH + query = f""" + SELECT id, asset, created, title, updated, topics, command, + (SELECT VALUE count() FROM source_insight WHERE source = $parent.id GROUP ALL)[0].count OR 0 AS insights_count, + (SELECT VALUE id FROM source_embedding WHERE source = $parent.id LIMIT 1) != [] AS embedded + FROM source + {order_clause} + LIMIT $limit START $offset + FETCH command + """ + result = await repo_query(query, {"limit": limit, "offset": offset}) + + # Convert result to response model + # Command data is already fetched via FETCH command clause + response_list = [] + for row in result: + command = row.get("command") + command_id = None + status = None + processing_info = None + + # Extract status from fetched command object (already resolved by FETCH) + if command and isinstance(command, dict): + command_id = str(command.get("id")) if command.get("id") else None + status = command.get("status") + # Extract execution metadata from nested result structure + result_data = command.get("result") + execution_metadata = ( + result_data.get("execution_metadata", {}) + if isinstance(result_data, dict) + else {} + ) + processing_info = { + "started_at": execution_metadata.get("started_at"), + "completed_at": execution_metadata.get("completed_at"), + "error": command.get("error_message"), + } + elif command: + # Command exists but FETCH failed to resolve it (broken reference) + command_id = str(command) + status = "unknown" + + response_list.append( + SourceListResponse( + id=row["id"], + title=row.get("title"), + topics=row.get("topics") or [], + asset=AssetModel( + file_path=row["asset"].get("file_path") + if row.get("asset") + else None, + url=row["asset"].get("url") if row.get("asset") else None, + ) + if row.get("asset") + else None, + embedded=row.get("embedded", False), + embedded_chunks=0, # Not needed in list view + insights_count=row.get("insights_count", 0), + created=str(row["created"]), + updated=str(row["updated"]), + # Status fields from fetched command + command_id=command_id, + status=status, + processing_info=processing_info, + ) + ) + + return response_list + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching sources: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching sources: {str(e)}") + + +@router.post("/sources", response_model=SourceResponse) +async def create_source( + form_data: tuple[SourceCreate, Optional[UploadFile]] = Depends( + parse_source_form_data + ), +): + """Create a new source with support for both JSON and multipart form data.""" + source_data, upload_file = form_data + + # Initialize file_path before try block so exception handlers can reference it + file_path = None + + try: + # Verify all specified notebooks exist (backward compatibility support) + for notebook_id in source_data.notebooks or []: + notebook = await Notebook.get(notebook_id) + if not notebook: + raise HTTPException( + status_code=404, detail=f"Notebook {notebook_id} not found" + ) + + # Handle file upload if provided + if upload_file and source_data.type == "upload": + try: + file_path = await save_uploaded_file(upload_file) + except Exception as e: + logger.error(f"File upload failed: {e}") + raise HTTPException( + status_code=400, detail=f"File upload failed: {str(e)}" + ) + + # Prepare content_state for processing + content_state: dict[str, Any] = {} + + if source_data.type == "link": + if not source_data.url: + raise HTTPException( + status_code=400, detail="URL is required for link type" + ) + content_state["url"] = source_data.url + elif source_data.type == "upload": + # Use uploaded file path or provided file_path (backward compatibility) + final_file_path = file_path or source_data.file_path + if not final_file_path: + raise HTTPException( + status_code=400, + detail="File upload or file_path is required for upload type", + ) + # Validate file_path is within the uploads directory to prevent LFI + uploads_resolved = Path(UPLOADS_FOLDER).resolve() + file_resolved = Path(final_file_path).resolve() + if not str(file_resolved).startswith(str(uploads_resolved) + os.sep): + raise HTTPException( + status_code=400, + detail="Invalid file path: must be within the uploads directory", + ) + content_state["file_path"] = final_file_path + content_state["delete_source"] = source_data.delete_source + elif source_data.type == "text": + if not source_data.content: + raise HTTPException( + status_code=400, detail="Content is required for text type" + ) + content_state["content"] = source_data.content + else: + raise HTTPException( + status_code=400, + detail="Invalid source type. Must be link, upload, or text", + ) + + # Validate transformations exist + transformation_ids = source_data.transformations or [] + for trans_id in transformation_ids: + transformation = await Transformation.get(trans_id) + if not transformation: + raise HTTPException( + status_code=404, detail=f"Transformation {trans_id} not found" + ) + + # Branch based on processing mode + if source_data.async_processing: + # ASYNC PATH: Create source record first, then queue command + logger.info("Using async processing path") + + # Create source record with asset - let SurrealDB generate the ID + # Persist asset before save so it's available for retry if processing fails + if source_data.type == "link": + source_asset = Asset(url=source_data.url) + elif source_data.type == "upload": + source_asset = Asset(file_path=file_path or source_data.file_path) + else: + source_asset = None + + source = Source( + title=source_data.title or "Processing...", + topics=[], + asset=source_asset, + ) + await source.save() + + # Add source to notebooks immediately so it appears in the UI + # The source_graph will skip adding duplicates + for notebook_id in source_data.notebooks or []: + await source.add_to_notebook(notebook_id) + + try: + # Import command modules to ensure they're registered + import commands.source_commands # noqa: F401 + + # Submit command for background processing + command_input = SourceProcessingInput( + source_id=str(source.id), + content_state=content_state, + notebook_ids=source_data.notebooks, + transformations=transformation_ids, + embed=source_data.embed, + ) + + command_id = await CommandService.submit_command_job( + "open_notebook", # app name + "process_source", # command name + command_input.model_dump(), + ) + + logger.info(f"Submitted async processing command: {command_id}") + + # Update source with command reference immediately + # command_id already includes 'command:' prefix + source.command = ensure_record_id(command_id) + await source.save() + + # Return source with command info + return SourceResponse( + id=source.id or "", + title=source.title, + topics=source.topics or [], + asset=None, # Will be populated after processing + full_text=None, # Will be populated after processing + embedded=False, # Will be updated after processing + embedded_chunks=0, + created=str(source.created), + updated=str(source.updated), + command_id=command_id, + status="new", + processing_info={"async": True, "queued": True}, + ) + + except Exception as e: + logger.error(f"Failed to submit async processing command: {e}") + # Clean up source record on command submission failure + try: + await source.delete() + except Exception: + pass + # Clean up uploaded file if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise HTTPException( + status_code=500, detail=f"Failed to queue processing: {str(e)}" + ) + + else: + # SYNC PATH: Execute synchronously using execute_command_sync + logger.info("Using sync processing path") + + try: + # Import command modules to ensure they're registered + import commands.source_commands # noqa: F401 + + # Create source record - let SurrealDB generate the ID + source = Source( + title=source_data.title or "Processing...", + topics=[], + ) + await source.save() + + # Add source to notebooks immediately so it appears in the UI + # The source_graph will skip adding duplicates + for notebook_id in source_data.notebooks or []: + await source.add_to_notebook(notebook_id) + + # Execute command synchronously + command_input = SourceProcessingInput( + source_id=str(source.id), + content_state=content_state, + notebook_ids=source_data.notebooks, + transformations=transformation_ids, + embed=source_data.embed, + ) + + # Run in thread pool to avoid blocking the event loop + # execute_command_sync uses asyncio.run() internally which can't + # be called from an already-running event loop (FastAPI) + result = await asyncio.to_thread( + execute_command_sync, + "open_notebook", # app name + "process_source", # command name + command_input.model_dump(), + timeout=300, # 5 minute timeout for sync processing + ) + + if not result.is_success(): + logger.error(f"Sync processing failed: {result.error_message}") + # Clean up source record + try: + await source.delete() + except Exception: + pass + # Clean up uploaded file if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise HTTPException( + status_code=500, + detail=f"Processing failed: {result.error_message}", + ) + + # Get the processed source + if not source.id: + raise HTTPException(status_code=500, detail="Source ID is missing") + processed_source = await Source.get(source.id) + if not processed_source: + raise HTTPException( + status_code=500, detail="Processed source not found" + ) + + embedded_chunks = await processed_source.get_embedded_chunks() + return SourceResponse( + id=processed_source.id or "", + title=processed_source.title, + topics=processed_source.topics or [], + asset=AssetModel( + file_path=processed_source.asset.file_path + if processed_source.asset + else None, + url=processed_source.asset.url + if processed_source.asset + else None, + ) + if processed_source.asset + else None, + full_text=processed_source.full_text, + embedded=embedded_chunks > 0, + embedded_chunks=embedded_chunks, + created=str(processed_source.created), + updated=str(processed_source.updated), + # No command_id or status for sync processing (legacy behavior) + ) + + except Exception as e: + logger.error(f"Sync processing failed: {e}") + # Clean up uploaded file if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise + + except HTTPException: + # Clean up uploaded file on HTTP exceptions if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise + except InvalidInputError as e: + # Clean up uploaded file on validation errors if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error creating source: {str(e)}") + # Clean up uploaded file on unexpected errors if we created it + if file_path and upload_file: + try: + os.unlink(file_path) + except Exception: + pass + raise HTTPException(status_code=500, detail=f"Error creating source: {str(e)}") + + +@router.post("/sources/json", response_model=SourceResponse) +async def create_source_json(source_data: SourceCreate): + """Create a new source using JSON payload (legacy endpoint for backward compatibility).""" + # Convert to form data format and call main endpoint + form_data = (source_data, None) + return await create_source(form_data) + + +async def _resolve_source_file(source_id: str) -> tuple[str, str]: + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + file_path = source.asset.file_path if source.asset else None + if not file_path: + raise HTTPException(status_code=404, detail="Source has no file to download") + + safe_root = os.path.realpath(UPLOADS_FOLDER) + resolved_path = os.path.realpath(file_path) + + if not resolved_path.startswith(safe_root): + logger.warning( + f"Blocked download outside uploads directory for source {source_id}: {resolved_path}" + ) + raise HTTPException(status_code=403, detail="Access to file denied") + + if not os.path.exists(resolved_path): + raise HTTPException(status_code=404, detail="File not found on server") + + filename = os.path.basename(resolved_path) + return resolved_path, filename + + +def _is_source_file_available(source: Source) -> Optional[bool]: + if not source or not source.asset or not source.asset.file_path: + return None + + file_path = source.asset.file_path + safe_root = os.path.realpath(UPLOADS_FOLDER) + resolved_path = os.path.realpath(file_path) + + if not resolved_path.startswith(safe_root): + return False + + return os.path.exists(resolved_path) + + +@router.get("/sources/{source_id}", response_model=SourceResponse) +async def get_source(source_id: str): + """Get a specific source by ID.""" + try: + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Get status information if command exists + status = None + processing_info = None + if source.command: + try: + status = await source.get_status() + processing_info = await source.get_processing_progress() + except Exception as e: + logger.warning(f"Failed to get status for source {source_id}: {e}") + status = "unknown" + + embedded_chunks = await source.get_embedded_chunks() + + # Get associated notebooks + notebooks_query = await repo_query( + "SELECT VALUE out FROM reference WHERE in = $source_id", + {"source_id": ensure_record_id(source.id or source_id)}, + ) + notebook_ids = ( + [str(nb_id) for nb_id in notebooks_query] if notebooks_query else [] + ) + + return SourceResponse( + id=source.id or "", + title=source.title, + topics=source.topics or [], + asset=AssetModel( + file_path=source.asset.file_path if source.asset else None, + url=source.asset.url if source.asset else None, + ) + if source.asset + else None, + full_text=source.full_text, + embedded=embedded_chunks > 0, + embedded_chunks=embedded_chunks, + file_available=_is_source_file_available(source), + created=str(source.created), + updated=str(source.updated), + # Status fields + command_id=str(source.command) if source.command else None, + status=status, + processing_info=processing_info, + # Notebook associations + notebooks=notebook_ids, + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching source {source_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error fetching source: {str(e)}") + + +@router.head("/sources/{source_id}/download") +async def check_source_file(source_id: str): + """Check if a source has a downloadable file.""" + try: + await _resolve_source_file(source_id) + return Response(status_code=200) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error checking file for source {source_id}: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to verify file") + + +@router.get("/sources/{source_id}/download") +async def download_source_file(source_id: str): + """Download the original file associated with an uploaded source.""" + try: + resolved_path, filename = await _resolve_source_file(source_id) + return FileResponse( + path=resolved_path, + filename=filename, + media_type="application/octet-stream", + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error downloading file for source {source_id}: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to download source file") + + +@router.get("/sources/{source_id}/status", response_model=SourceStatusResponse) +async def get_source_status(source_id: str): + """Get processing status for a source.""" + try: + # First, verify source exists + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Check if this is a legacy source (no command) + if not source.command: + return SourceStatusResponse( + status=None, + message="Legacy source (completed before async processing)", + processing_info=None, + command_id=None, + ) + + # Get command status and processing info + try: + status = await source.get_status() + processing_info = await source.get_processing_progress() + + # Generate descriptive message based on status + if status == "completed": + message = "Source processing completed successfully" + elif status == "failed": + message = "Source processing failed" + elif status == "running": + message = "Source processing in progress" + elif status == "queued": + message = "Source processing queued" + elif status == "unknown": + message = "Source processing status unknown" + else: + message = f"Source processing status: {status}" + + return SourceStatusResponse( + status=status, + message=message, + processing_info=processing_info, + command_id=str(source.command) if source.command else None, + ) + + except Exception as e: + logger.warning(f"Failed to get status for source {source_id}: {e}") + return SourceStatusResponse( + status="unknown", + message="Failed to retrieve processing status", + processing_info=None, + command_id=str(source.command) if source.command else None, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching status for source {source_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching source status: {str(e)}" + ) + + +@router.put("/sources/{source_id}", response_model=SourceResponse) +async def update_source(source_id: str, source_update: SourceUpdate): + """Update a source.""" + try: + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Update only provided fields + if source_update.title is not None: + source.title = source_update.title + if source_update.topics is not None: + source.topics = source_update.topics + + await source.save() + + embedded_chunks = await source.get_embedded_chunks() + return SourceResponse( + id=source.id or "", + title=source.title, + topics=source.topics or [], + asset=AssetModel( + file_path=source.asset.file_path if source.asset else None, + url=source.asset.url if source.asset else None, + ) + if source.asset + else None, + full_text=source.full_text, + embedded=embedded_chunks > 0, + embedded_chunks=embedded_chunks, + created=str(source.created), + updated=str(source.updated), + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error updating source {source_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error updating source: {str(e)}") + + +@router.post("/sources/{source_id}/retry", response_model=SourceResponse) +async def retry_source_processing(source_id: str): + """Retry processing for a failed or stuck source.""" + try: + # First, verify source exists + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Check if source already has a running command + if source.command: + try: + status = await source.get_status() + if status in ["running", "queued"]: + raise HTTPException( + status_code=400, + detail="Source is already processing. Cannot retry while processing is active.", + ) + except Exception as e: + logger.warning( + f"Failed to check current status for source {source_id}: {e}" + ) + # Continue with retry if we can't check status + + # Get notebooks that this source belongs to + query = "SELECT notebook FROM reference WHERE source = $source_id" + references = await repo_query(query, {"source_id": source_id}) + notebook_ids = [str(ref["notebook"]) for ref in references] + + if not notebook_ids: + raise HTTPException( + status_code=400, detail="Source is not associated with any notebooks" + ) + + # Prepare content_state based on source asset + content_state = {} + if source.asset: + if source.asset.file_path: + content_state = { + "file_path": source.asset.file_path, + "delete_source": False, # Don't delete on retry + } + elif source.asset.url: + content_state = {"url": source.asset.url} + else: + raise HTTPException( + status_code=400, detail="Source asset has no file_path or url" + ) + else: + # Check if it's a text source by trying to get full_text + if source.full_text: + content_state = {"content": source.full_text} + else: + raise HTTPException( + status_code=400, detail="Cannot determine source content for retry" + ) + + try: + # Import command modules to ensure they're registered + import commands.source_commands # noqa: F401 + + # Submit new command for background processing + command_input = SourceProcessingInput( + source_id=str(source.id), + content_state=content_state, + notebook_ids=notebook_ids, + transformations=[], # Use default transformations on retry + embed=True, # Always embed on retry + ) + + command_id = await CommandService.submit_command_job( + "open_notebook", # app name + "process_source", # command name + command_input.model_dump(), + ) + + logger.info( + f"Submitted retry processing command: {command_id} for source {source_id}" + ) + + # Update source with new command ID + source.command = ensure_record_id(f"command:{command_id}") + await source.save() + + # Get current embedded chunks count + embedded_chunks = await source.get_embedded_chunks() + + # Return updated source response + return SourceResponse( + id=source.id or "", + title=source.title, + topics=source.topics or [], + asset=AssetModel( + file_path=source.asset.file_path if source.asset else None, + url=source.asset.url if source.asset else None, + ) + if source.asset + else None, + full_text=source.full_text, + embedded=embedded_chunks > 0, + embedded_chunks=embedded_chunks, + created=str(source.created), + updated=str(source.updated), + command_id=command_id, + status="queued", + processing_info={"retry": True, "queued": True}, + ) + + except Exception as e: + logger.error( + f"Failed to submit retry processing command for source {source_id}: {e}" + ) + raise HTTPException( + status_code=500, detail=f"Failed to queue retry processing: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error retrying source processing for {source_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error retrying source processing: {str(e)}" + ) + + +@router.delete("/sources/{source_id}") +async def delete_source(source_id: str): + """Delete a source.""" + try: + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + await source.delete() + + return {"message": "Source deleted successfully"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting source {source_id}: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error deleting source: {str(e)}") + + +@router.get("/sources/{source_id}/insights", response_model=List[SourceInsightResponse]) +async def get_source_insights(source_id: str): + """Get all insights for a specific source.""" + try: + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + insights = await source.get_insights() + return [ + SourceInsightResponse( + id=insight.id or "", + source_id=source_id, + insight_type=insight.insight_type, + content=insight.content, + created=str(insight.created), + updated=str(insight.updated), + ) + for insight in insights + ] + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching insights for source {source_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching insights: {str(e)}" + ) + + +@router.post( + "/sources/{source_id}/insights", + response_model=InsightCreationResponse, + status_code=202, +) +async def create_source_insight(source_id: str, request: CreateSourceInsightRequest): + """ + Start insight generation for a source by running a transformation. + + This endpoint returns immediately with a 202 Accepted status. + The transformation runs asynchronously in the background via the job queue. + Poll GET /sources/{source_id}/insights to see when the insight is ready. + """ + try: + # Validate source exists + source = await Source.get(source_id) + if not source: + raise HTTPException(status_code=404, detail="Source not found") + + # Validate transformation exists + transformation = await Transformation.get(request.transformation_id) + if not transformation: + raise HTTPException(status_code=404, detail="Transformation not found") + + # Submit transformation as background job (fire-and-forget) + command_id = submit_command( + "open_notebook", + "run_transformation", + { + "source_id": source_id, + "transformation_id": request.transformation_id, + }, + ) + logger.info( + f"Submitted run_transformation command {command_id} for source {source_id}" + ) + + # Return immediately with command_id for status tracking + return InsightCreationResponse( + status="pending", + message="Insight generation started", + source_id=source_id, + transformation_id=request.transformation_id, + command_id=str(command_id), + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error starting insight generation for source {source_id}: {e}") + raise HTTPException( + status_code=500, detail=f"Error starting insight generation: {str(e)}" + ) diff --git a/api/routers/speaker_profiles.py b/api/routers/speaker_profiles.py new file mode 100644 index 0000000000000000000000000000000000000000..a44016002edf667cad29f40ade19a06991171c17 --- /dev/null +++ b/api/routers/speaker_profiles.py @@ -0,0 +1,190 @@ +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException +from loguru import logger +from pydantic import BaseModel, Field + +from open_notebook.podcasts.models import SpeakerProfile + +router = APIRouter() + + +class SpeakerProfileResponse(BaseModel): + id: str + name: str + description: str + voice_model: Optional[str] = None + speakers: List[Dict[str, Any]] + # Legacy fields (for display/migration awareness) + tts_provider: Optional[str] = None + tts_model: Optional[str] = None + + +def _profile_to_response(profile: SpeakerProfile) -> SpeakerProfileResponse: + return SpeakerProfileResponse( + id=str(profile.id), + name=profile.name, + description=profile.description or "", + voice_model=profile.voice_model, + speakers=profile.speakers, + tts_provider=profile.tts_provider, + tts_model=profile.tts_model, + ) + + +@router.get("/speaker-profiles", response_model=List[SpeakerProfileResponse]) +async def list_speaker_profiles(): + """List all available speaker profiles""" + try: + profiles = await SpeakerProfile.get_all(order_by="name asc") + return [_profile_to_response(p) for p in profiles] + except Exception as e: + logger.error(f"Failed to fetch speaker profiles: {e}") + raise HTTPException( + status_code=500, detail="Failed to fetch speaker profiles" + ) + + +@router.get("/speaker-profiles/{profile_name}", response_model=SpeakerProfileResponse) +async def get_speaker_profile(profile_name: str): + """Get a specific speaker profile by name""" + try: + profile = await SpeakerProfile.get_by_name(profile_name) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Speaker profile '{profile_name}' not found" + ) + + return _profile_to_response(profile) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to fetch speaker profile '{profile_name}': {e}") + raise HTTPException( + status_code=500, detail="Failed to fetch speaker profile" + ) + + +class SpeakerProfileCreate(BaseModel): + name: str = Field(..., description="Unique profile name") + description: str = Field("", description="Profile description") + voice_model: Optional[str] = Field(None, description="Model record ID for TTS") + speakers: List[Dict[str, Any]] = Field( + ..., description="Array of speaker configurations" + ) + # Legacy fields (accepted but not required) + tts_provider: Optional[str] = None + tts_model: Optional[str] = None + + +@router.post("/speaker-profiles", response_model=SpeakerProfileResponse) +async def create_speaker_profile(profile_data: SpeakerProfileCreate): + """Create a new speaker profile""" + try: + profile = SpeakerProfile( + name=profile_data.name, + description=profile_data.description, + voice_model=profile_data.voice_model, + speakers=profile_data.speakers, + tts_provider=profile_data.tts_provider, + tts_model=profile_data.tts_model, + ) + + await profile.save() + return _profile_to_response(profile) + + except Exception as e: + logger.error(f"Failed to create speaker profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to create speaker profile" + ) + + +@router.put("/speaker-profiles/{profile_id}", response_model=SpeakerProfileResponse) +async def update_speaker_profile(profile_id: str, profile_data: SpeakerProfileCreate): + """Update an existing speaker profile""" + try: + profile = await SpeakerProfile.get(profile_id) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Speaker profile '{profile_id}' not found" + ) + + profile.name = profile_data.name + profile.description = profile_data.description + profile.voice_model = profile_data.voice_model + profile.speakers = profile_data.speakers + profile.tts_provider = profile_data.tts_provider + profile.tts_model = profile_data.tts_model + + await profile.save() + return _profile_to_response(profile) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update speaker profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to update speaker profile" + ) + + +@router.delete("/speaker-profiles/{profile_id}") +async def delete_speaker_profile(profile_id: str): + """Delete a speaker profile""" + try: + profile = await SpeakerProfile.get(profile_id) + + if not profile: + raise HTTPException( + status_code=404, detail=f"Speaker profile '{profile_id}' not found" + ) + + await profile.delete() + + return {"message": "Speaker profile deleted successfully"} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to delete speaker profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to delete speaker profile" + ) + + +@router.post( + "/speaker-profiles/{profile_id}/duplicate", response_model=SpeakerProfileResponse +) +async def duplicate_speaker_profile(profile_id: str): + """Duplicate a speaker profile""" + try: + original = await SpeakerProfile.get(profile_id) + + if not original: + raise HTTPException( + status_code=404, detail=f"Speaker profile '{profile_id}' not found" + ) + + duplicate = SpeakerProfile( + name=f"{original.name} - Copy", + description=original.description, + voice_model=original.voice_model, + speakers=original.speakers, + tts_provider=original.tts_provider, + tts_model=original.tts_model, + ) + + await duplicate.save() + return _profile_to_response(duplicate) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to duplicate speaker profile: {e}") + raise HTTPException( + status_code=500, detail="Failed to duplicate speaker profile" + ) diff --git a/api/routers/transformations.py b/api/routers/transformations.py new file mode 100644 index 0000000000000000000000000000000000000000..1848ad54b0b3f20f98b83d9a705b0be82421b92d --- /dev/null +++ b/api/routers/transformations.py @@ -0,0 +1,252 @@ +from typing import List + +from fastapi import APIRouter, HTTPException +from loguru import logger + +from api.models import ( + DefaultPromptResponse, + DefaultPromptUpdate, + TransformationCreate, + TransformationExecuteRequest, + TransformationExecuteResponse, + TransformationResponse, + TransformationUpdate, +) +from open_notebook.ai.models import Model +from open_notebook.domain.transformation import DefaultPrompts, Transformation +from open_notebook.exceptions import InvalidInputError, OpenNotebookError +from open_notebook.graphs.transformation import graph as transformation_graph + +router = APIRouter() + + +@router.get("/transformations", response_model=List[TransformationResponse]) +async def get_transformations(): + """Get all transformations.""" + try: + transformations = await Transformation.get_all(order_by="name asc") + + return [ + TransformationResponse( + id=transformation.id or "", + name=transformation.name, + title=transformation.title, + description=transformation.description, + prompt=transformation.prompt, + apply_default=transformation.apply_default, + created=str(transformation.created), + updated=str(transformation.updated), + ) + for transformation in transformations + ] + except Exception as e: + logger.error(f"Error fetching transformations: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching transformations: {str(e)}" + ) + + +@router.post("/transformations", response_model=TransformationResponse) +async def create_transformation(transformation_data: TransformationCreate): + """Create a new transformation.""" + try: + new_transformation = Transformation( + name=transformation_data.name, + title=transformation_data.title, + description=transformation_data.description, + prompt=transformation_data.prompt, + apply_default=transformation_data.apply_default, + ) + await new_transformation.save() + + return TransformationResponse( + id=new_transformation.id or "", + name=new_transformation.name, + title=new_transformation.title, + description=new_transformation.description, + prompt=new_transformation.prompt, + apply_default=new_transformation.apply_default, + created=str(new_transformation.created), + updated=str(new_transformation.updated), + ) + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error creating transformation: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error creating transformation: {str(e)}" + ) + + +@router.post("/transformations/execute", response_model=TransformationExecuteResponse) +async def execute_transformation(execute_request: TransformationExecuteRequest): + """Execute a transformation on input text.""" + try: + # Validate transformation exists + transformation = await Transformation.get(execute_request.transformation_id) + if not transformation: + raise HTTPException(status_code=404, detail="Transformation not found") + + # Validate model exists + model = await Model.get(execute_request.model_id) + if not model: + raise HTTPException(status_code=404, detail="Model not found") + + # Execute the transformation + result = await transformation_graph.ainvoke( + dict( # type: ignore[arg-type] + input_text=execute_request.input_text, + transformation=transformation, + ), + config=dict(configurable={"model_id": execute_request.model_id}), + ) + + return TransformationExecuteResponse( + output=result["output"], + transformation_id=execute_request.transformation_id, + model_id=execute_request.model_id, + ) + + except HTTPException: + raise + except OpenNotebookError: + raise # Let global exception handlers return proper status codes + except Exception as e: + logger.error(f"Error executing transformation: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error executing transformation: {str(e)}" + ) + + +@router.get("/transformations/default-prompt", response_model=DefaultPromptResponse) +async def get_default_prompt(): + """Get the default transformation prompt.""" + try: + default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment] + + return DefaultPromptResponse( + transformation_instructions=default_prompts.transformation_instructions + or "" + ) + except Exception as e: + logger.error(f"Error fetching default prompt: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching default prompt: {str(e)}" + ) + + +@router.put("/transformations/default-prompt", response_model=DefaultPromptResponse) +async def update_default_prompt(prompt_update: DefaultPromptUpdate): + """Update the default transformation prompt.""" + try: + default_prompts: DefaultPrompts = await DefaultPrompts.get_instance() # type: ignore[assignment] + + default_prompts.transformation_instructions = ( + prompt_update.transformation_instructions + ) + await default_prompts.update() + + return DefaultPromptResponse( + transformation_instructions=default_prompts.transformation_instructions + ) + except Exception as e: + logger.error(f"Error updating default prompt: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error updating default prompt: {str(e)}" + ) + + +@router.get( + "/transformations/{transformation_id}", response_model=TransformationResponse +) +async def get_transformation(transformation_id: str): + """Get a specific transformation by ID.""" + try: + transformation = await Transformation.get(transformation_id) + if not transformation: + raise HTTPException(status_code=404, detail="Transformation not found") + + return TransformationResponse( + id=transformation.id or "", + name=transformation.name, + title=transformation.title, + description=transformation.description, + prompt=transformation.prompt, + apply_default=transformation.apply_default, + created=str(transformation.created), + updated=str(transformation.updated), + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error fetching transformation {transformation_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error fetching transformation: {str(e)}" + ) + + +@router.put( + "/transformations/{transformation_id}", response_model=TransformationResponse +) +async def update_transformation( + transformation_id: str, transformation_update: TransformationUpdate +): + """Update a transformation.""" + try: + transformation = await Transformation.get(transformation_id) + if not transformation: + raise HTTPException(status_code=404, detail="Transformation not found") + + # Update only provided fields + if transformation_update.name is not None: + transformation.name = transformation_update.name + if transformation_update.title is not None: + transformation.title = transformation_update.title + if transformation_update.description is not None: + transformation.description = transformation_update.description + if transformation_update.prompt is not None: + transformation.prompt = transformation_update.prompt + if transformation_update.apply_default is not None: + transformation.apply_default = transformation_update.apply_default + + await transformation.save() + + return TransformationResponse( + id=transformation.id or "", + name=transformation.name, + title=transformation.title, + description=transformation.description, + prompt=transformation.prompt, + apply_default=transformation.apply_default, + created=str(transformation.created), + updated=str(transformation.updated), + ) + except HTTPException: + raise + except InvalidInputError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Error updating transformation {transformation_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error updating transformation: {str(e)}" + ) + + +@router.delete("/transformations/{transformation_id}") +async def delete_transformation(transformation_id: str): + """Delete a transformation.""" + try: + transformation = await Transformation.get(transformation_id) + if not transformation: + raise HTTPException(status_code=404, detail="Transformation not found") + + await transformation.delete() + + return {"message": "Transformation deleted successfully"} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting transformation {transformation_id}: {str(e)}") + raise HTTPException( + status_code=500, detail=f"Error deleting transformation: {str(e)}" + ) diff --git a/api/search_service.py b/api/search_service.py new file mode 100644 index 0000000000000000000000000000000000000000..57e0580c1487945491c9d6464e61cff974989df6 --- /dev/null +++ b/api/search_service.py @@ -0,0 +1,58 @@ +""" +Search service layer using API. +""" + +from typing import Any, Dict, List, Union + +from loguru import logger + +from api.client import api_client + + +class SearchService: + """Service layer for search operations using API.""" + + def __init__(self): + logger.info("Using API for search operations") + + def search( + self, + query: str, + search_type: str = "text", + limit: int = 100, + search_sources: bool = True, + search_notes: bool = True, + minimum_score: float = 0.2, + ) -> List[Dict[str, Any]]: + """Search the knowledge base.""" + response = api_client.search( + query=query, + search_type=search_type, + limit=limit, + search_sources=search_sources, + search_notes=search_notes, + minimum_score=minimum_score, + ) + if isinstance(response, dict): + return response.get("results", []) + return [] + + def ask_knowledge_base( + self, + question: str, + strategy_model: str, + answer_model: str, + final_answer_model: str, + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Ask the knowledge base a question.""" + response = api_client.ask_simple( + question=question, + strategy_model=strategy_model, + answer_model=answer_model, + final_answer_model=final_answer_model, + ) + return response + + +# Global service instance +search_service = SearchService() diff --git a/api/settings_service.py b/api/settings_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e8a43f56a2002b810b29de9fc0917edac53688ab --- /dev/null +++ b/api/settings_service.py @@ -0,0 +1,79 @@ +""" +Settings service layer using API. +""" + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.content_settings import ContentSettings + + +class SettingsService: + """Service layer for settings operations using API.""" + + def __init__(self): + logger.info("Using API for settings operations") + + def get_settings(self) -> ContentSettings: + """Get application settings.""" + settings_response = api_client.get_settings() + settings_data = ( + settings_response + if isinstance(settings_response, dict) + else settings_response[0] + ) + + # Create ContentSettings object from API response + settings = ContentSettings( + default_content_processing_engine_doc=settings_data.get( + "default_content_processing_engine_doc" + ), + default_content_processing_engine_url=settings_data.get( + "default_content_processing_engine_url" + ), + default_embedding_option=settings_data.get("default_embedding_option"), + auto_delete_files=settings_data.get("auto_delete_files"), + youtube_preferred_languages=settings_data.get( + "youtube_preferred_languages" + ), + ) + + return settings + + def update_settings(self, settings: ContentSettings) -> ContentSettings: + """Update application settings.""" + updates = { + "default_content_processing_engine_doc": settings.default_content_processing_engine_doc, + "default_content_processing_engine_url": settings.default_content_processing_engine_url, + "default_embedding_option": settings.default_embedding_option, + "auto_delete_files": settings.auto_delete_files, + "youtube_preferred_languages": settings.youtube_preferred_languages, + } + + settings_response = api_client.update_settings(**updates) + settings_data = ( + settings_response + if isinstance(settings_response, dict) + else settings_response[0] + ) + + # Update the settings object with the response + settings.default_content_processing_engine_doc = settings_data.get( + "default_content_processing_engine_doc" + ) + settings.default_content_processing_engine_url = settings_data.get( + "default_content_processing_engine_url" + ) + settings.default_embedding_option = settings_data.get( + "default_embedding_option" + ) + settings.auto_delete_files = settings_data.get("auto_delete_files") + settings.youtube_preferred_languages = settings_data.get( + "youtube_preferred_languages" + ) + + return settings + + +# Global service instance +settings_service = SettingsService() diff --git a/api/sources_service.py b/api/sources_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ae03375c1a0c0d3c6c2034a2eb929171df103265 --- /dev/null +++ b/api/sources_service.py @@ -0,0 +1,324 @@ +""" +Sources service layer using API. +""" + +from dataclasses import dataclass +from typing import Dict, List, Optional, Union + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.notebook import Asset, Source + + +@dataclass +class SourceProcessingResult: + """Result of source creation with optional async processing info.""" + + source: Source + is_async: bool = False + command_id: Optional[str] = None + status: Optional[str] = None + processing_info: Optional[Dict] = None + + +@dataclass +class SourceWithMetadata: + """Source object with additional metadata from API.""" + + source: Source + embedded_chunks: int + + # Expose common source properties for easy access + @property + def id(self): + return self.source.id + + @property + def title(self): + return self.source.title + + @title.setter + def title(self, value): + self.source.title = value + + @property + def topics(self): + return self.source.topics + + @property + def asset(self): + return self.source.asset + + @property + def full_text(self): + return self.source.full_text + + @property + def created(self): + return self.source.created + + @property + def updated(self): + return self.source.updated + + +class SourcesService: + """Service layer for sources operations using API.""" + + def __init__(self): + logger.info("Using API for sources operations") + + def get_all_sources( + self, notebook_id: Optional[str] = None + ) -> List[SourceWithMetadata]: + """Get all sources with optional notebook filtering.""" + sources_data = api_client.get_sources(notebook_id=notebook_id) + # Convert API response to SourceWithMetadata objects + sources = [] + for source_data in sources_data: + source = Source( + title=source_data["title"], + topics=source_data["topics"], + asset=Asset( + file_path=source_data["asset"]["file_path"] + if source_data["asset"] + else None, + url=source_data["asset"]["url"] if source_data["asset"] else None, + ) + if source_data["asset"] + else None, + ) + source.id = source_data["id"] + source.created = source_data["created"] + source.updated = source_data["updated"] + + # Wrap in SourceWithMetadata + source_with_metadata = SourceWithMetadata( + source=source, embedded_chunks=source_data.get("embedded_chunks", 0) + ) + sources.append(source_with_metadata) + return sources + + def get_source(self, source_id: str) -> SourceWithMetadata: + """Get a specific source.""" + response = api_client.get_source(source_id) + source_data = response if isinstance(response, dict) else response[0] + source = Source( + title=source_data["title"], + topics=source_data["topics"], + full_text=source_data["full_text"], + asset=Asset( + file_path=source_data["asset"]["file_path"] + if source_data["asset"] + else None, + url=source_data["asset"]["url"] if source_data["asset"] else None, + ) + if source_data["asset"] + else None, + ) + source.id = source_data["id"] + source.created = source_data["created"] + source.updated = source_data["updated"] + + return SourceWithMetadata( + source=source, embedded_chunks=source_data.get("embedded_chunks", 0) + ) + + def create_source( + self, + notebook_id: Optional[str] = None, + source_type: str = "text", + url: Optional[str] = None, + file_path: Optional[str] = None, + content: Optional[str] = None, + title: Optional[str] = None, + transformations: Optional[List[str]] = None, + embed: bool = False, + delete_source: bool = False, + notebooks: Optional[List[str]] = None, + async_processing: bool = False, + ) -> Union[Source, SourceProcessingResult]: + """ + Create a new source with support for async processing. + + Args: + notebook_id: Single notebook ID (deprecated, use notebooks parameter) + source_type: Type of source (link, upload, text) + url: URL for link sources + file_path: File path for upload sources + content: Text content for text sources + title: Optional source title + transformations: List of transformation IDs to apply + embed: Whether to embed content for vector search + delete_source: Whether to delete uploaded file after processing + notebooks: List of notebook IDs to add source to (preferred over notebook_id) + async_processing: Whether to process source asynchronously + + Returns: + Source object for sync processing (backward compatibility) + SourceProcessingResult for async processing (contains additional metadata) + """ + source_data = api_client.create_source( + notebook_id=notebook_id, + notebooks=notebooks, + source_type=source_type, + url=url, + file_path=file_path, + content=content, + title=title, + transformations=transformations, + embed=embed, + delete_source=delete_source, + async_processing=async_processing, + ) + + # Create Source object from response + response_data = source_data if isinstance(source_data, dict) else source_data[0] + source = Source( + title=response_data["title"], + topics=response_data.get("topics") or [], + full_text=response_data.get("full_text"), + asset=Asset( + file_path=response_data["asset"]["file_path"] + if response_data.get("asset") + else None, + url=response_data["asset"]["url"] + if response_data.get("asset") + else None, + ) + if response_data.get("asset") + else None, + ) + source.id = response_data["id"] + source.created = response_data["created"] + source.updated = response_data["updated"] + + # Check if this is an async processing response + if ( + response_data.get("command_id") + or response_data.get("status") + or response_data.get("processing_info") + ): + # Ensure source_data is a dict for accessing attributes + source_data_dict = ( + source_data if isinstance(source_data, dict) else source_data[0] + ) + # Return enhanced result for async processing + return SourceProcessingResult( + source=source, + is_async=True, + command_id=source_data_dict.get("command_id"), + status=source_data_dict.get("status"), + processing_info=source_data_dict.get("processing_info"), + ) + else: + # Return simple Source for backward compatibility + return source + + def get_source_status(self, source_id: str) -> Dict: + """Get processing status for a source.""" + response = api_client.get_source_status(source_id) + return response if isinstance(response, dict) else response[0] + + def create_source_async( + self, + notebook_id: Optional[str] = None, + source_type: str = "text", + url: Optional[str] = None, + file_path: Optional[str] = None, + content: Optional[str] = None, + title: Optional[str] = None, + transformations: Optional[List[str]] = None, + embed: bool = False, + delete_source: bool = False, + notebooks: Optional[List[str]] = None, + ) -> SourceProcessingResult: + """ + Create a new source with async processing enabled. + + This is a convenience method that always uses async processing. + Returns a SourceProcessingResult with processing status information. + """ + result = self.create_source( + notebook_id=notebook_id, + notebooks=notebooks, + source_type=source_type, + url=url, + file_path=file_path, + content=content, + title=title, + transformations=transformations, + embed=embed, + delete_source=delete_source, + async_processing=True, + ) + + # Since we forced async_processing=True, this should always be a SourceProcessingResult + if isinstance(result, SourceProcessingResult): + return result + else: + # Fallback: wrap Source in SourceProcessingResult + return SourceProcessingResult( + source=result, + is_async=False, # This shouldn't happen, but handle it gracefully + ) + + def is_source_processing_complete(self, source_id: str) -> bool: + """ + Check if a source's async processing is complete. + + Returns True if processing is complete (success or failure), + False if still processing or queued. + """ + try: + status_data = self.get_source_status(source_id) + status = status_data.get("status") + return status in [ + "completed", + "failed", + None, + ] # None indicates legacy/sync source + except Exception as e: + logger.error(f"Error checking source processing status: {e}") + return True # Assume complete on error + + def update_source(self, source: Source) -> Source: + """Update a source.""" + if not source.id: + raise ValueError("Source ID is required for update") + + updates = { + "title": source.title, + "topics": source.topics, + } + source_data = api_client.update_source(source.id, **updates) + + # Ensure source_data is a dict + source_data_dict = ( + source_data if isinstance(source_data, dict) else source_data[0] + ) + + # Update the source object with the response + source.title = source_data_dict["title"] + source.topics = source_data_dict["topics"] + source.updated = source_data_dict["updated"] + + return source + + def delete_source(self, source_id: str) -> bool: + """Delete a source.""" + api_client.delete_source(source_id) + return True + + +# Global service instance +sources_service = SourcesService() + +# Export important classes for easy importing +__all__ = [ + "SourcesService", + "SourceWithMetadata", + "SourceProcessingResult", + "sources_service", +] diff --git a/api/transformations_service.py b/api/transformations_service.py new file mode 100644 index 0000000000000000000000000000000000000000..bc81c08c50b5818afd1018cf1aba815bf8e72e7c --- /dev/null +++ b/api/transformations_service.py @@ -0,0 +1,141 @@ +""" +Transformations service layer using API. +""" + +from datetime import datetime +from typing import Any, Dict, List, Union + +from loguru import logger + +from api.client import api_client +from open_notebook.domain.transformation import Transformation + + +class TransformationsService: + """Service layer for transformations operations using API.""" + + def __init__(self): + logger.info("Using API for transformations operations") + + def get_all_transformations(self) -> List[Transformation]: + """Get all transformations.""" + transformations_data = api_client.get_transformations() + # Convert API response to Transformation objects + transformations = [] + for trans_data in transformations_data: + transformation = Transformation( + name=trans_data["name"], + title=trans_data["title"], + description=trans_data["description"], + prompt=trans_data["prompt"], + apply_default=trans_data["apply_default"], + ) + transformation.id = trans_data["id"] + transformation.created = datetime.fromisoformat( + trans_data["created"].replace("Z", "+00:00") + ) + transformation.updated = datetime.fromisoformat( + trans_data["updated"].replace("Z", "+00:00") + ) + transformations.append(transformation) + return transformations + + def get_transformation(self, transformation_id: str) -> Transformation: + """Get a specific transformation.""" + response = api_client.get_transformation(transformation_id) + trans_data = response if isinstance(response, dict) else response[0] + transformation = Transformation( + name=trans_data["name"], + title=trans_data["title"], + description=trans_data["description"], + prompt=trans_data["prompt"], + apply_default=trans_data["apply_default"], + ) + transformation.id = trans_data["id"] + transformation.created = datetime.fromisoformat( + trans_data["created"].replace("Z", "+00:00") + ) + transformation.updated = datetime.fromisoformat( + trans_data["updated"].replace("Z", "+00:00") + ) + return transformation + + def create_transformation( + self, + name: str, + title: str, + description: str, + prompt: str, + apply_default: bool = False, + ) -> Transformation: + """Create a new transformation.""" + response = api_client.create_transformation( + name=name, + title=title, + description=description, + prompt=prompt, + apply_default=apply_default, + ) + trans_data = response if isinstance(response, dict) else response[0] + transformation = Transformation( + name=trans_data["name"], + title=trans_data["title"], + description=trans_data["description"], + prompt=trans_data["prompt"], + apply_default=trans_data["apply_default"], + ) + transformation.id = trans_data["id"] + transformation.created = datetime.fromisoformat( + trans_data["created"].replace("Z", "+00:00") + ) + transformation.updated = datetime.fromisoformat( + trans_data["updated"].replace("Z", "+00:00") + ) + return transformation + + def update_transformation(self, transformation: Transformation) -> Transformation: + """Update a transformation.""" + if not transformation.id: + raise ValueError("Transformation ID is required for update") + + updates = { + "name": transformation.name, + "title": transformation.title, + "description": transformation.description, + "prompt": transformation.prompt, + "apply_default": transformation.apply_default, + } + response = api_client.update_transformation(transformation.id, **updates) + trans_data = response if isinstance(response, dict) else response[0] + + # Update the transformation object with the response + transformation.name = trans_data["name"] + transformation.title = trans_data["title"] + transformation.description = trans_data["description"] + transformation.prompt = trans_data["prompt"] + transformation.apply_default = trans_data["apply_default"] + transformation.updated = datetime.fromisoformat( + trans_data["updated"].replace("Z", "+00:00") + ) + + return transformation + + def delete_transformation(self, transformation_id: str) -> bool: + """Delete a transformation.""" + api_client.delete_transformation(transformation_id) + return True + + def execute_transformation( + self, transformation_id: str, input_text: str, model_id: str + ) -> Union[Dict[Any, Any], List[Dict[Any, Any]]]: + """Execute a transformation on input text.""" + result = api_client.execute_transformation( + transformation_id=transformation_id, + input_text=input_text, + model_id=model_id, + ) + return result + + +# Global service instance +transformations_service = TransformationsService() diff --git a/commands/CLAUDE.md b/commands/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..708e8436040b0f34056a3d8012a4e7b852c07366 --- /dev/null +++ b/commands/CLAUDE.md @@ -0,0 +1,68 @@ +# Commands Module + +**Purpose**: Defines async command handlers for long-running operations via `surreal-commands` job queue system. + +## Key Components + +### Embedding Commands + +- **`embed_note_command`**: Embeds a single note using unified embedding pipeline with content-type aware processing. Uses MARKDOWN content type detection. Retry: 5 attempts, exponential jitter 1-60s. +- **`embed_insight_command`**: Embeds a single source insight. Uses MARKDOWN content type. Retry: 5 attempts, exponential jitter 1-60s. +- **`embed_source_command`**: Embeds a source by chunking full_text with content-type aware splitters (HTML, Markdown, plain), then batch embedding all chunks (batches of 50 with per-batch retry). Retry: 5 attempts, exponential jitter 1-60s. +- **`create_insight_command`**: Creates a source insight with automatic retry on transaction conflicts. Creates the DB record, then submits `embed_insight` command (fire-and-forget). Retry: 5 attempts, exponential jitter 1-60s. Used by `Source.add_insight()`. +- **`rebuild_embeddings_command`**: Submits individual embed_* commands for all sources/notes/insights. Returns immediately; actual embedding happens async. No retry (coordinator only). + +### Other Commands + +- **`process_source_command`**: Ingests content through `source_graph`, creates embeddings (optional), and generates insights. Retries on transaction conflicts (exp. jitter, max 15×, 1-120s). +- **`run_transformation_command`**: Runs a transformation on an existing source to generate an insight. Executes the transformation graph (LLM call) then creates insight via `create_insight_command`. Used by `POST /sources/{id}/insights` API endpoint. Retry: 5 attempts, exponential jitter 1-60s. +- **`generate_podcast_command`**: Creates podcasts via podcast-creator library. Resolves model registry references and credentials for all profiles before invoking podcast-creator. Validates that outline_llm, transcript_llm, and voice_model are configured. +- **`process_text_command`** (example): Test fixture for text operations (uppercase, lowercase, reverse, word_count). +- **`analyze_data_command`** (example): Test fixture for numeric aggregations. + +## Important Patterns + +- **Pydantic I/O**: All commands use `CommandInput`/`CommandOutput` subclasses for type safety and serialization. +- **Error handling**: Permanent errors (ValueError) return failure output; all other exceptions auto-retry via surreal-commands. +- **Retry configuration**: Uses `stop_on: [ValueError]` (blocklist approach) - retries all exceptions EXCEPT ValueError. This is more resilient than allowlist as new exception types auto-retry. +- **Fire-and-forget embedding**: Domain models submit embed_* commands via `submit_command()` without waiting. Commands process asynchronously. +- **Content-type aware chunking**: `embed_source_command` uses `chunk_text()` with automatic content type detection (HTML, Markdown, plain text) for optimal text splitting. Default: 1500 char chunks with 225 char overlap. +- **Batch embedding**: `embed_source_command` uses `generate_embeddings()` which automatically batches texts (default 50) with per-batch retry to avoid exceeding provider payload limits. +- **Mean pooling for large content**: `embed_note_command` and `embed_insight_command` use `generate_embedding()` which handles content larger than chunk size via mean pooling. +- **Model dumping**: Recursive `full_model_dump()` utility converts Pydantic models → dicts for DB/API responses. +- **Logging**: Uses `loguru.logger` throughout; logs execution start/end and key metrics (processing time, counts). +- **Time tracking**: All commands measure `start_time` → `processing_time` for monitoring. + +## Dependencies + +**External**: `surreal_commands` (command decorator, job queue, submit_command), `loguru`, `pydantic`, `podcast_creator` +**Internal**: `open_notebook.domain.notebook` (Source, Note, SourceInsight), `open_notebook.utils.chunking` (chunk_text, detect_content_type), `open_notebook.utils.embedding` (generate_embedding, generate_embeddings), `open_notebook.database.repository` (repo_query, repo_insert) + +## Quirks & Edge Cases + +- **source_commands**: `ensure_record_id()` wraps command IDs for DB storage; transaction conflicts trigger exponential backoff retry. ValueError exceptions are permanent (not retried). +- **embedding_commands**: Content type detection uses file extension as primary source, heuristics as fallback. Chunks >1800 chars trigger secondary splitting. Empty/whitespace-only content returns ValueError (not retried). +- **rebuild_embeddings_command**: Returns "jobs_submitted" not "processed_items" - embedding is async. Individual commands handle failures with their own retries. +- **podcast_commands**: Profiles loaded from SurrealDB by name; model configs (credentials) resolved for ALL profiles before podcast-creator validation. Validates outline_llm/transcript_llm/voice_model are set. Episode records created mid-execution. +- **Example commands**: Accept optional `delay_seconds` for testing async behavior; not for production. + +## Code Example + +```python +@command("process_source", app="open_notebook", retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "stop_on": [ValueError], # Don't retry validation errors +}) +async def process_source_command(input_data: SourceProcessingInput) -> SourceProcessingOutput: + start_time = time.time() + try: + transformations = [await Transformation.get(id) for id in input_data.transformations] + source = await Source.get(input_data.source_id) + result = await source_graph.ainvoke({...}) + return SourceProcessingOutput(success=True, ...) + except ValueError as e: + return SourceProcessingOutput(success=False, error_message=str(e)) # No retry + except Exception as e: + raise # Retry all other exceptions +``` diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..479b952cb196e2b771f60faa0db50ffefbd6cb38 --- /dev/null +++ b/commands/__init__.py @@ -0,0 +1,24 @@ +"""Surreal-commands integration for Open Notebook""" + +from .embedding_commands import ( + embed_insight_command, + embed_note_command, + embed_source_command, + rebuild_embeddings_command, +) +from .example_commands import analyze_data_command, process_text_command +from .podcast_commands import generate_podcast_command +from .source_commands import process_source_command + +__all__ = [ + # Embedding commands + "embed_note_command", + "embed_insight_command", + "embed_source_command", + "rebuild_embeddings_command", + # Other commands + "generate_podcast_command", + "process_source_command", + "process_text_command", + "analyze_data_command", +] diff --git a/commands/embedding_commands.py b/commands/embedding_commands.py new file mode 100644 index 0000000000000000000000000000000000000000..f6ec70d302e20db903505d35ab5c066630cf4180 --- /dev/null +++ b/commands/embedding_commands.py @@ -0,0 +1,787 @@ +import time +from typing import Dict, List, Literal, Optional + +from loguru import logger +from pydantic import BaseModel +from surreal_commands import CommandInput, CommandOutput, command, submit_command + +from open_notebook.ai.models import model_manager +from open_notebook.database.repository import ensure_record_id, repo_insert, repo_query +from open_notebook.exceptions import ConfigurationError +from open_notebook.domain.notebook import Note, Source, SourceInsight +from open_notebook.utils.chunking import ContentType, chunk_text, detect_content_type +from open_notebook.utils.embedding import generate_embedding, generate_embeddings + + +def full_model_dump(model): + if isinstance(model, BaseModel): + return model.model_dump() + elif isinstance(model, dict): + return {k: full_model_dump(v) for k, v in model.items()} + elif isinstance(model, list): + return [full_model_dump(item) for item in model] + else: + return model + + +def get_command_id(input_data: CommandInput) -> str: + """Extract command_id from input_data's execution context, or return 'unknown'.""" + if input_data.execution_context: + return str(input_data.execution_context.command_id) + return "unknown" + + +class RebuildEmbeddingsInput(CommandInput): + mode: Literal["existing", "all"] + include_sources: bool = True + include_notes: bool = True + include_insights: bool = True + + +class RebuildEmbeddingsOutput(CommandOutput): + success: bool + total_items: int + jobs_submitted: int # Count of embedding commands submitted + failed_submissions: int # Count of items that failed to submit + sources_submitted: int = 0 + notes_submitted: int = 0 + insights_submitted: int = 0 + processing_time: float + error_message: Optional[str] = None + + +# ============================================================================= +# NEW EMBEDDING COMMANDS (Phase 3) +# ============================================================================= + + +class CreateInsightInput(CommandInput): + """Input for creating a source insight with automatic retry on conflicts.""" + + source_id: str + insight_type: str + content: str + + +class CreateInsightOutput(CommandOutput): + """Output from insight creation command.""" + + success: bool + insight_id: Optional[str] = None + processing_time: float + error_message: Optional[str] = None + + +class EmbedNoteInput(CommandInput): + """Input for embedding a single note.""" + + note_id: str + + +class EmbedNoteOutput(CommandOutput): + """Output from note embedding command.""" + + success: bool + note_id: str + processing_time: float + error_message: Optional[str] = None + + +class EmbedInsightInput(CommandInput): + """Input for embedding a single source insight.""" + + insight_id: str + + +class EmbedInsightOutput(CommandOutput): + """Output from insight embedding command.""" + + success: bool + insight_id: str + processing_time: float + error_message: Optional[str] = None + + +class EmbedSourceInput(CommandInput): + """Input for embedding a source (creates multiple chunk embeddings).""" + + source_id: str + + +class EmbedSourceOutput(CommandOutput): + """Output from source embedding command.""" + + success: bool + source_id: str + chunks_created: int + processing_time: float + error_message: Optional[str] = None + + +@command( + "embed_note", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 60, + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", + }, +) +async def embed_note_command(input_data: EmbedNoteInput) -> EmbedNoteOutput: + """ + Generate and store embedding for a single note. + + Uses the unified embedding pipeline with automatic chunking and mean pooling + for notes that exceed the chunk size limit. + + Flow: + 1. Load Note by ID + 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed) + 3. UPSERT note embedding in database + + Retry Strategy: + - Retries up to 5 times for transient failures (network, timeout, etc.) + - Uses exponential-jitter backoff (1-60s) + - Does NOT retry permanent failures (ValueError for validation errors) + """ + start_time = time.time() + + try: + logger.info(f"Starting embedding for note: {input_data.note_id}") + + # 1. Load note + note = await Note.get(input_data.note_id) + if not note: + raise ValueError(f"Note '{input_data.note_id}' not found") + + if not note.content or not note.content.strip(): + raise ValueError(f"Note '{input_data.note_id}' has no content to embed") + + # 2. Generate embedding (auto-chunks + mean pools if needed) + # Notes are typically markdown content + cmd_id = get_command_id(input_data) + embedding = await generate_embedding( + note.content, content_type=ContentType.MARKDOWN, command_id=cmd_id + ) + + # 3. UPSERT embedding into note record + await repo_query( + "UPDATE $note_id SET embedding = $embedding", + { + "note_id": ensure_record_id(input_data.note_id), + "embedding": embedding, + }, + ) + + processing_time = time.time() - start_time + logger.info( + f"Successfully embedded note {input_data.note_id} in {processing_time:.2f}s" + ) + + return EmbedNoteOutput( + success=True, + note_id=input_data.note_id, + processing_time=processing_time, + ) + + except ValueError as e: + # Permanent failure - don't retry + processing_time = time.time() - start_time + cmd_id = get_command_id(input_data) + logger.error( + f"Failed to embed note {input_data.note_id} (command: {cmd_id}): {e}" + ) + return EmbedNoteOutput( + success=False, + note_id=input_data.note_id, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + cmd_id = get_command_id(input_data) + logger.debug( + f"Transient error embedding note {input_data.note_id} " + f"(command: {cmd_id}): {e}" + ) + raise + + +@command( + "embed_insight", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 60, + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", + }, +) +async def embed_insight_command(input_data: EmbedInsightInput) -> EmbedInsightOutput: + """ + Generate and store embedding for a single source insight. + + Uses the unified embedding pipeline with automatic chunking and mean pooling + for insights that exceed the chunk size limit. + + Flow: + 1. Load SourceInsight by ID + 2. Generate embedding via generate_embedding() (auto-chunks + mean pools if needed) + 3. UPSERT insight embedding in database + + Retry Strategy: + - Retries up to 5 times for transient failures (network, timeout, etc.) + - Uses exponential-jitter backoff (1-60s) + - Does NOT retry permanent failures (ValueError for validation errors) + """ + start_time = time.time() + + try: + logger.info(f"Starting embedding for insight: {input_data.insight_id}") + + # 1. Load insight + insight = await SourceInsight.get(input_data.insight_id) + if not insight: + raise ValueError(f"Insight '{input_data.insight_id}' not found") + + if not insight.content or not insight.content.strip(): + raise ValueError( + f"Insight '{input_data.insight_id}' has no content to embed" + ) + + # 2. Generate embedding (auto-chunks + mean pools if needed) + # Insights are typically markdown content (generated by LLM) + cmd_id = get_command_id(input_data) + embedding = await generate_embedding( + insight.content, content_type=ContentType.MARKDOWN, command_id=cmd_id + ) + + # 3. UPSERT embedding into insight record + await repo_query( + "UPDATE $insight_id SET embedding = $embedding", + { + "insight_id": ensure_record_id(input_data.insight_id), + "embedding": embedding, + }, + ) + + processing_time = time.time() - start_time + logger.info( + f"Successfully embedded insight {input_data.insight_id} in {processing_time:.2f}s" + ) + + return EmbedInsightOutput( + success=True, + insight_id=input_data.insight_id, + processing_time=processing_time, + ) + + except ValueError as e: + # Permanent failure - don't retry + processing_time = time.time() - start_time + cmd_id = get_command_id(input_data) + logger.error( + f"Failed to embed insight {input_data.insight_id} (command: {cmd_id}): {e}" + ) + return EmbedInsightOutput( + success=False, + insight_id=input_data.insight_id, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + cmd_id = get_command_id(input_data) + logger.debug( + f"Transient error embedding insight {input_data.insight_id} " + f"(command: {cmd_id}): {e}" + ) + raise + + +@command( + "embed_source", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 60, + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", + }, +) +async def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutput: + """ + Generate and store embeddings for a source document. + + Creates multiple chunk embeddings stored in the source_embedding table. + Uses content-type aware chunking based on file extension or content heuristics. + + Flow: + 1. Load Source by ID + 2. DELETE existing source_embedding records for this source + 3. Detect content type from file path or content + 4. Chunk text using appropriate splitter + 5. Generate embeddings for all chunks in batches + 6. Bulk INSERT source_embedding records + + Retry Strategy: + - Retries up to 5 times for transient failures (network, timeout, etc.) + - Uses exponential-jitter backoff (1-60s) + - Does NOT retry permanent failures (ValueError for validation errors) + """ + start_time = time.time() + + try: + logger.info(f"Starting embedding for source: {input_data.source_id}") + + # 1. Load source + source = await Source.get(input_data.source_id) + if not source: + raise ValueError(f"Source '{input_data.source_id}' not found") + + if not source.full_text or not source.full_text.strip(): + raise ValueError(f"Source '{input_data.source_id}' has no text to embed") + + # 2. DELETE existing embeddings (idempotency) + logger.debug(f"Deleting existing embeddings for source {input_data.source_id}") + await repo_query( + "DELETE source_embedding WHERE source = $source_id", + {"source_id": ensure_record_id(input_data.source_id)}, + ) + + # 3. Detect content type from file path if available + file_path = source.asset.file_path if source.asset else None + content_type = detect_content_type(source.full_text, file_path) + logger.debug(f"Detected content type: {content_type.value}") + + # 4. Chunk text using appropriate splitter + chunks = chunk_text(source.full_text, content_type=content_type) + total_chunks = len(chunks) + + # Log chunk statistics for debugging + chunk_sizes = [len(c) for c in chunks] + logger.info( + f"Created {total_chunks} chunks for source {input_data.source_id} " + f"(sizes: min={min(chunk_sizes) if chunk_sizes else 0}, " + f"max={max(chunk_sizes) if chunk_sizes else 0}, " + f"avg={sum(chunk_sizes)//len(chunk_sizes) if chunk_sizes else 0} chars)" + ) + + if total_chunks == 0: + raise ValueError("No chunks created after splitting text") + + # 5. Generate embeddings for all chunks in batches + cmd_id = get_command_id(input_data) + logger.debug(f"Generating embeddings for {total_chunks} chunks") + embeddings = await generate_embeddings(chunks, command_id=cmd_id) + + # Verify we got embeddings for all chunks + if len(embeddings) != len(chunks): + raise ValueError( + f"Embedding count mismatch: got {len(embeddings)} embeddings " + f"for {len(chunks)} chunks" + ) + + # 6. Bulk INSERT source_embedding records + records = [ + { + "source": ensure_record_id(input_data.source_id), + "order": idx, + "content": chunk, + "embedding": embedding, + } + for idx, (chunk, embedding) in enumerate(zip(chunks, embeddings)) + ] + + logger.debug(f"Inserting {len(records)} source_embedding records") + await repo_insert("source_embedding", records) + + processing_time = time.time() - start_time + logger.info( + f"Successfully embedded source {input_data.source_id}: " + f"{total_chunks} chunks in {processing_time:.2f}s" + ) + + return EmbedSourceOutput( + success=True, + source_id=input_data.source_id, + chunks_created=total_chunks, + processing_time=processing_time, + ) + + except ValueError as e: + # Permanent failure - don't retry + processing_time = time.time() - start_time + cmd_id = get_command_id(input_data) + logger.error( + f"Failed to embed source {input_data.source_id} (command: {cmd_id}): {e}" + ) + return EmbedSourceOutput( + success=False, + source_id=input_data.source_id, + chunks_created=0, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + cmd_id = get_command_id(input_data) + logger.debug( + f"Transient error embedding source {input_data.source_id} " + f"(command: {cmd_id}): {e}" + ) + raise + + +@command( + "create_insight", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 60, + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", + }, +) +async def create_insight_command( + input_data: CreateInsightInput, +) -> CreateInsightOutput: + """ + Create a source insight with automatic retry on transaction conflicts. + + This command wraps the CREATE source_insight operation with retry logic + to handle SurrealDB transaction conflicts that occur during batch imports + when multiple parallel transformations try to create insights concurrently. + + Flow: + 1. CREATE source_insight record in database + 2. Submit embed_insight command (fire-and-forget) for async embedding + 3. Return the insight_id + + Retry Strategy: + - Retries up to 5 times for transient failures (network, timeout, etc.) + - Uses exponential-jitter backoff (1-60s) + - Does NOT retry permanent failures (ValueError for validation errors) + """ + start_time = time.time() + + try: + logger.info( + f"Creating insight for source {input_data.source_id}: " + f"type={input_data.insight_type}" + ) + + # 1. Create insight record in database + result = await repo_query( + """ + CREATE source_insight CONTENT { + "source": $source_id, + "insight_type": $insight_type, + "content": $content + }; + """, + { + "source_id": ensure_record_id(input_data.source_id), + "insight_type": input_data.insight_type, + "content": input_data.content, + }, + ) + + if not result or len(result) == 0: + raise ValueError("Failed to create insight - no result returned") + + insight_id = str(result[0].get("id", "")) + if not insight_id: + raise ValueError("Failed to create insight - no ID in result") + + # 2. Submit embedding command (fire-and-forget) + submit_command( + "open_notebook", + "embed_insight", + {"insight_id": insight_id}, + ) + logger.debug(f"Submitted embed_insight command for {insight_id}") + + processing_time = time.time() - start_time + logger.info( + f"Successfully created insight {insight_id} for source " + f"{input_data.source_id} in {processing_time:.2f}s" + ) + + return CreateInsightOutput( + success=True, + insight_id=insight_id, + processing_time=processing_time, + ) + + except ValueError as e: + # Permanent failure - don't retry + processing_time = time.time() - start_time + cmd_id = get_command_id(input_data) + logger.error( + f"Failed to create insight for source {input_data.source_id} " + f"(command: {cmd_id}): {e}" + ) + return CreateInsightOutput( + success=False, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + cmd_id = get_command_id(input_data) + logger.debug( + f"Transient error creating insight for source {input_data.source_id} " + f"(command: {cmd_id}): {e}" + ) + raise + + +async def collect_items_for_rebuild( + mode: str, + include_sources: bool, + include_notes: bool, + include_insights: bool, +) -> Dict[str, List[str]]: + """ + Collect items to rebuild based on mode and include flags. + + Returns: + Dict with keys: 'sources', 'notes', 'insights' containing lists of item IDs + """ + items: Dict[str, List[str]] = {"sources": [], "notes": [], "insights": []} + + if include_sources: + if mode == "existing": + # Query sources with embeddings (via source_embedding table) + result = await repo_query( + """ + RETURN array::distinct( + SELECT VALUE source.id + FROM source_embedding + WHERE embedding != none AND array::len(embedding) > 0 + ) + """ + ) + # RETURN returns the array directly as the result (not nested) + if result: + items["sources"] = [str(item) for item in result] + else: + items["sources"] = [] + else: # mode == "all" + # Query all sources with non-empty content + result = await repo_query( + "SELECT id FROM source WHERE full_text != none AND string::trim(full_text) != ''" + ) + items["sources"] = [str(item["id"]) for item in result] if result else [] + + logger.info(f"Collected {len(items['sources'])} sources for rebuild") + + if include_notes: + if mode == "existing": + # Query notes with embeddings + result = await repo_query( + "SELECT id FROM note WHERE embedding != none AND array::len(embedding) > 0" + ) + else: # mode == "all" + # Query all notes with non-empty content + result = await repo_query( + "SELECT id FROM note WHERE content != none AND string::trim(content) != ''" + ) + + items["notes"] = [str(item["id"]) for item in result] if result else [] + logger.info(f"Collected {len(items['notes'])} notes for rebuild") + + if include_insights: + if mode == "existing": + # Query insights with embeddings + result = await repo_query( + "SELECT id FROM source_insight WHERE embedding != none AND array::len(embedding) > 0" + ) + else: # mode == "all" + # Query all insights with non-empty content + result = await repo_query( + "SELECT id FROM source_insight WHERE content != none AND string::trim(content) != ''" + ) + + items["insights"] = [str(item["id"]) for item in result] if result else [] + logger.info(f"Collected {len(items['insights'])} insights for rebuild") + + return items + + +@command("rebuild_embeddings", app="open_notebook", retry=None) +async def rebuild_embeddings_command( + input_data: RebuildEmbeddingsInput, +) -> RebuildEmbeddingsOutput: + """ + Rebuild embeddings for sources, notes, and/or insights. + + This command submits individual embedding jobs for each item: + - embed_source for sources + - embed_note for notes + - embed_insight for insights + + The command returns after submitting all jobs. Actual embedding + happens asynchronously via the individual commands (which have + their own retry strategies). + + Retry Strategy: + - Retries disabled (retry=None) for this coordinator command + - Individual embed_* commands handle their own retries + """ + start_time = time.time() + + try: + logger.info("=" * 60) + logger.info(f"Starting embedding rebuild with mode={input_data.mode}") + logger.info( + f"Include: sources={input_data.include_sources}, notes={input_data.include_notes}, insights={input_data.include_insights}" + ) + logger.info("=" * 60) + + # Check embedding model availability (fail fast) + EMBEDDING_MODEL = await model_manager.get_embedding_model() + if not EMBEDDING_MODEL: + raise ValueError( + "No embedding model configured. Please configure one in the Models section." + ) + + logger.info(f"Embedding model configured: {EMBEDDING_MODEL}") + + # Collect items to process (returns IDs only) + items = await collect_items_for_rebuild( + input_data.mode, + input_data.include_sources, + input_data.include_notes, + input_data.include_insights, + ) + + total_items = ( + len(items["sources"]) + len(items["notes"]) + len(items["insights"]) + ) + logger.info(f"Total items to rebuild: {total_items}") + + if total_items == 0: + logger.warning("No items found to rebuild") + return RebuildEmbeddingsOutput( + success=True, + total_items=0, + jobs_submitted=0, + failed_submissions=0, + processing_time=time.time() - start_time, + ) + + # Initialize counters + sources_submitted = 0 + notes_submitted = 0 + insights_submitted = 0 + failed_submissions = 0 + + # Submit embed_source commands for sources + logger.info(f"\nSubmitting {len(items['sources'])} source embedding jobs...") + for idx, source_id in enumerate(items["sources"], 1): + try: + submit_command( + "open_notebook", + "embed_source", + {"source_id": source_id}, + ) + sources_submitted += 1 + + if idx % 50 == 0 or idx == len(items["sources"]): + logger.info( + f" Progress: {idx}/{len(items['sources'])} source jobs submitted" + ) + + except Exception as e: + logger.error(f"Failed to submit embed_source for {source_id}: {e}") + failed_submissions += 1 + + # Submit embed_note commands for notes + logger.info(f"\nSubmitting {len(items['notes'])} note embedding jobs...") + for idx, note_id in enumerate(items["notes"], 1): + try: + submit_command( + "open_notebook", + "embed_note", + {"note_id": note_id}, + ) + notes_submitted += 1 + + if idx % 50 == 0 or idx == len(items["notes"]): + logger.info( + f" Progress: {idx}/{len(items['notes'])} note jobs submitted" + ) + + except Exception as e: + logger.error(f"Failed to submit embed_note for {note_id}: {e}") + failed_submissions += 1 + + # Submit embed_insight commands for insights + logger.info(f"\nSubmitting {len(items['insights'])} insight embedding jobs...") + for idx, insight_id in enumerate(items["insights"], 1): + try: + submit_command( + "open_notebook", + "embed_insight", + {"insight_id": insight_id}, + ) + insights_submitted += 1 + + if idx % 50 == 0 or idx == len(items["insights"]): + logger.info( + f" Progress: {idx}/{len(items['insights'])} insight jobs submitted" + ) + + except Exception as e: + logger.error(f"Failed to submit embed_insight for {insight_id}: {e}") + failed_submissions += 1 + + processing_time = time.time() - start_time + jobs_submitted = sources_submitted + notes_submitted + insights_submitted + + logger.info("=" * 60) + logger.info("REBUILD JOBS SUBMITTED") + logger.info(f" Total jobs submitted: {jobs_submitted}/{total_items}") + logger.info(f" Sources: {sources_submitted}") + logger.info(f" Notes: {notes_submitted}") + logger.info(f" Insights: {insights_submitted}") + logger.info(f" Failed submissions: {failed_submissions}") + logger.info(f" Submission time: {processing_time:.2f}s") + logger.info(" Note: Actual embedding happens asynchronously") + logger.info("=" * 60) + + return RebuildEmbeddingsOutput( + success=True, + total_items=total_items, + jobs_submitted=jobs_submitted, + failed_submissions=failed_submissions, + sources_submitted=sources_submitted, + notes_submitted=notes_submitted, + insights_submitted=insights_submitted, + processing_time=processing_time, + ) + + except Exception as e: + processing_time = time.time() - start_time + logger.error(f"Rebuild embeddings failed: {e}") + logger.exception(e) + + return RebuildEmbeddingsOutput( + success=False, + total_items=0, + jobs_submitted=0, + failed_submissions=0, + processing_time=processing_time, + error_message=str(e), + ) diff --git a/commands/example_commands.py b/commands/example_commands.py new file mode 100644 index 0000000000000000000000000000000000000000..7f26cb988db7d113721207b666acabbac4eba19d --- /dev/null +++ b/commands/example_commands.py @@ -0,0 +1,142 @@ +import asyncio +import time +from typing import List, Optional + +from loguru import logger +from pydantic import BaseModel +from surreal_commands import command + + +class TextProcessingInput(BaseModel): + text: str + operation: str = "uppercase" # uppercase, lowercase, word_count, reverse + delay_seconds: Optional[int] = None # For testing async behavior + + +class TextProcessingOutput(BaseModel): + success: bool + original_text: str + processed_text: Optional[str] = None + word_count: Optional[int] = None + processing_time: float + error_message: Optional[str] = None + + +class DataAnalysisInput(BaseModel): + numbers: List[float] + analysis_type: str = "basic" # basic, detailed + delay_seconds: Optional[int] = None + + +class DataAnalysisOutput(BaseModel): + success: bool + analysis_type: str + count: int + sum: Optional[float] = None + average: Optional[float] = None + min_value: Optional[float] = None + max_value: Optional[float] = None + processing_time: float + error_message: Optional[str] = None + + +@command("process_text", app="open_notebook") +async def process_text_command(input_data: TextProcessingInput) -> TextProcessingOutput: + """ + Example command for text processing. Tests basic command functionality + and demonstrates different processing types. + """ + start_time = time.time() + + try: + logger.info(f"Processing text with operation: {input_data.operation}") + + # Simulate processing delay if specified + if input_data.delay_seconds: + await asyncio.sleep(input_data.delay_seconds) + + processed_text = None + word_count = None + + if input_data.operation == "uppercase": + processed_text = input_data.text.upper() + elif input_data.operation == "lowercase": + processed_text = input_data.text.lower() + elif input_data.operation == "reverse": + processed_text = input_data.text[::-1] + elif input_data.operation == "word_count": + word_count = len(input_data.text.split()) + processed_text = f"Word count: {word_count}" + else: + raise ValueError(f"Unknown operation: {input_data.operation}") + + processing_time = time.time() - start_time + + return TextProcessingOutput( + success=True, + original_text=input_data.text, + processed_text=processed_text, + word_count=word_count, + processing_time=processing_time, + ) + + except Exception as e: + processing_time = time.time() - start_time + logger.error(f"Text processing failed: {e}") + return TextProcessingOutput( + success=False, + original_text=input_data.text, + processing_time=processing_time, + error_message=str(e), + ) + + +@command("analyze_data", app="open_notebook") +async def analyze_data_command(input_data: DataAnalysisInput) -> DataAnalysisOutput: + """ + Example command for data analysis. Tests command with complex input/output + and demonstrates error handling. + """ + start_time = time.time() + + try: + logger.info( + f"Analyzing {len(input_data.numbers)} numbers with {input_data.analysis_type} analysis" + ) + + # Simulate processing delay if specified + if input_data.delay_seconds: + await asyncio.sleep(input_data.delay_seconds) + + if not input_data.numbers: + raise ValueError("No numbers provided for analysis") + + count = len(input_data.numbers) + sum_value = sum(input_data.numbers) + average = sum_value / count + min_value = min(input_data.numbers) + max_value = max(input_data.numbers) + + processing_time = time.time() - start_time + + return DataAnalysisOutput( + success=True, + analysis_type=input_data.analysis_type, + count=count, + sum=sum_value, + average=average, + min_value=min_value, + max_value=max_value, + processing_time=processing_time, + ) + + except Exception as e: + processing_time = time.time() - start_time + logger.error(f"Data analysis failed: {e}") + return DataAnalysisOutput( + success=False, + analysis_type=input_data.analysis_type, + count=0, + processing_time=processing_time, + error_message=str(e), + ) diff --git a/commands/podcast_commands.py b/commands/podcast_commands.py new file mode 100644 index 0000000000000000000000000000000000000000..34ffae85c6a1b407f505a374f7e234d3133416ca --- /dev/null +++ b/commands/podcast_commands.py @@ -0,0 +1,300 @@ +import time +import uuid +from pathlib import Path +from typing import Optional + +from loguru import logger +from pydantic import BaseModel +from surreal_commands import CommandInput, CommandOutput, command + +from open_notebook.config import DATA_FOLDER +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.podcasts.models import ( + EpisodeProfile, + PodcastEpisode, + SpeakerProfile, + _resolve_model_config, +) + +try: + from podcast_creator import configure, create_podcast +except ImportError as e: + logger.error(f"Failed to import podcast_creator: {e}") + raise ValueError("podcast_creator library not available") + + +def build_episode_output_dir(data_folder: str) -> tuple[str, Path]: + """Build a filesystem-safe output directory path for a podcast episode. + + Uses a UUID as the directory name so the path is safe regardless of + what the user typed as episode name (spaces, special chars, etc.). + + Returns: + A tuple of (episode_dir_name, output_dir_path). + """ + episode_dir_name = str(uuid.uuid4()) + output_dir = Path(f"{data_folder}/podcasts/episodes/{episode_dir_name}") + return episode_dir_name, output_dir + + +def full_model_dump(model): + if isinstance(model, BaseModel): + return model.model_dump() + elif isinstance(model, dict): + return {k: full_model_dump(v) for k, v in model.items()} + elif isinstance(model, list): + return [full_model_dump(item) for item in model] + else: + return model + + +class PodcastGenerationInput(CommandInput): + episode_profile: str + speaker_profile: str + episode_name: str + content: str + briefing_suffix: Optional[str] = None + + +class PodcastGenerationOutput(CommandOutput): + success: bool + episode_id: Optional[str] = None + audio_file_path: Optional[str] = None + transcript: Optional[dict] = None + outline: Optional[dict] = None + processing_time: float + error_message: Optional[str] = None + + +@command("generate_podcast", app="open_notebook", retry={"max_attempts": 1}) +async def generate_podcast_command( + input_data: PodcastGenerationInput, +) -> PodcastGenerationOutput: + """ + Real podcast generation using podcast-creator library with Episode Profiles + """ + start_time = time.time() + + try: + logger.info( + f"Starting podcast generation for episode: {input_data.episode_name}" + ) + logger.info(f"Using episode profile: {input_data.episode_profile}") + + # 1. Load Episode and Speaker profiles from SurrealDB + episode_profile = await EpisodeProfile.get_by_name(input_data.episode_profile) + if not episode_profile: + raise ValueError( + f"Episode profile '{input_data.episode_profile}' not found" + ) + + speaker_profile = await SpeakerProfile.get_by_name( + episode_profile.speaker_config + ) + if not speaker_profile: + raise ValueError( + f"Speaker profile '{episode_profile.speaker_config}' not found" + ) + + logger.info(f"Loaded episode profile: {episode_profile.name}") + logger.info(f"Loaded speaker profile: {speaker_profile.name}") + + # 2. Validate that model registry fields are populated + if not episode_profile.outline_llm: + raise ValueError( + f"Episode profile '{episode_profile.name}' has no outline model configured. " + "Please update the profile to select an outline model." + ) + if not episode_profile.transcript_llm: + raise ValueError( + f"Episode profile '{episode_profile.name}' has no transcript model configured. " + "Please update the profile to select a transcript model." + ) + if not speaker_profile.voice_model: + raise ValueError( + f"Speaker profile '{speaker_profile.name}' has no voice model configured. " + "Please update the profile to select a voice model." + ) + + # 3. Resolve model configs with credentials + outline_provider, outline_model_name, outline_config = ( + await episode_profile.resolve_outline_config() + ) + transcript_provider, transcript_model_name, transcript_config = ( + await episode_profile.resolve_transcript_config() + ) + tts_provider, tts_model_name, tts_config = ( + await speaker_profile.resolve_tts_config() + ) + + logger.info( + f"Resolved models - outline: {outline_provider}/{outline_model_name}, " + f"transcript: {transcript_provider}/{transcript_model_name}, " + f"tts: {tts_provider}/{tts_model_name}" + ) + + # 4. Load all profiles and configure podcast-creator + episode_profiles = await repo_query("SELECT * FROM episode_profile") + speaker_profiles = await repo_query("SELECT * FROM speaker_profile") + + # Transform the surrealdb array into a dictionary for podcast-creator + episode_profiles_dict = { + profile["name"]: profile for profile in episode_profiles + } + speaker_profiles_dict = { + profile["name"]: profile for profile in speaker_profiles + } + + # 5. Inject resolved model configs into profile dicts + # Resolve ALL episode profiles (podcast-creator validates all). + # Remove profiles that fail resolution to prevent validation errors. + for ep_name in list(episode_profiles_dict.keys()): + ep_dict = episode_profiles_dict[ep_name] + try: + if ep_dict.get("outline_llm"): + prov, model, conf = await _resolve_model_config( + str(ep_dict["outline_llm"]) + ) + ep_dict["outline_provider"] = prov + ep_dict["outline_model"] = model + ep_dict["outline_config"] = conf + if ep_dict.get("transcript_llm"): + prov, model, conf = await _resolve_model_config( + str(ep_dict["transcript_llm"]) + ) + ep_dict["transcript_provider"] = prov + ep_dict["transcript_model"] = model + ep_dict["transcript_config"] = conf + except Exception as e: + logger.warning( + f"Failed to resolve models for episode profile '{ep_name}', " + f"removing from config to prevent validation errors: {e}" + ) + del episode_profiles_dict[ep_name] + + # Resolve TTS for ALL speaker profiles (podcast-creator validates all). + # Remove profiles that fail resolution to prevent validation errors. + for sp_name in list(speaker_profiles_dict.keys()): + sp_dict = speaker_profiles_dict[sp_name] + if sp_dict.get("voice_model"): + try: + prov, model, conf = await _resolve_model_config( + str(sp_dict["voice_model"]) + ) + sp_dict["tts_provider"] = prov + sp_dict["tts_model"] = model + sp_dict["tts_config"] = conf + except Exception as e: + logger.warning( + f"Failed to resolve TTS for speaker profile '{sp_name}', " + f"removing from config to prevent validation errors: {e}" + ) + del speaker_profiles_dict[sp_name] + continue + + # Per-speaker TTS overrides + for speaker in sp_dict.get("speakers", []): + if speaker.get("voice_model"): + try: + prov, model, conf = await _resolve_model_config( + str(speaker["voice_model"]) + ) + speaker["tts_provider"] = prov + speaker["tts_model"] = model + speaker["tts_config"] = conf + except Exception as e: + logger.warning( + f"Failed to resolve per-speaker TTS for '{speaker.get('name')}': {e}" + ) + + # 6. Generate briefing + briefing = episode_profile.default_briefing + if input_data.briefing_suffix: + briefing += f"\n\nAdditional instructions: {input_data.briefing_suffix}" + + # Create the record for the episode and associate with the ongoing command + episode = PodcastEpisode( + name=input_data.episode_name, + episode_profile=full_model_dump(episode_profile.model_dump()), + speaker_profile=full_model_dump(speaker_profile.model_dump()), + command=ensure_record_id(input_data.execution_context.command_id) + if input_data.execution_context + else None, + briefing=briefing, + content=input_data.content, + audio_file=None, + transcript=None, + outline=None, + ) + await episode.save() + + configure("speakers_config", {"profiles": speaker_profiles_dict}) + configure("episode_config", {"profiles": episode_profiles_dict}) + + logger.info("Configured podcast-creator with episode and speaker profiles") + + logger.info(f"Generated briefing (length: {len(briefing)} chars)") + + # 7. Create output directory using UUID for filesystem-safe paths + episode_dir_name, output_dir = build_episode_output_dir(DATA_FOLDER) + output_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Created output directory: {output_dir}") + + # 8. Generate podcast using podcast-creator + logger.info("Starting podcast generation with podcast-creator...") + + result = await create_podcast( + content=input_data.content, + briefing=briefing, + episode_name=episode_dir_name, + output_dir=str(output_dir), + speaker_config=speaker_profile.name, + episode_profile=episode_profile.name, + ) + + episode.audio_file = ( + str(result.get("final_output_file_path")) if result else None + ) + episode.transcript = { + "transcript": full_model_dump(result["transcript"]) if result else None + } + episode.outline = full_model_dump(result["outline"]) if result else None + await episode.save() + + processing_time = time.time() - start_time + logger.info( + f"Successfully generated podcast episode: {episode.id} in {processing_time:.2f}s" + ) + + return PodcastGenerationOutput( + success=True, + episode_id=str(episode.id), + audio_file_path=str(result.get("final_output_file_path")) + if result + else None, + transcript={"transcript": full_model_dump(result["transcript"])} + if result.get("transcript") + else None, + outline=full_model_dump(result["outline"]) + if result.get("outline") + else None, + processing_time=processing_time, + ) + + except ValueError: + raise + + except Exception as e: + logger.error(f"Podcast generation failed: {e}") + logger.exception(e) + + error_msg = str(e) + if "Invalid json output" in error_msg or "Expecting value" in error_msg: + error_msg += ( + "\n\nNOTE: This error commonly occurs with GPT-5 models that use extended thinking. " + "The model may be putting all output inside tags, leaving nothing to parse. " + "Try using gpt-4o, gpt-4o-mini, or gpt-4-turbo instead in your episode profile." + ) + + raise RuntimeError(error_msg) from e diff --git a/commands/source_commands.py b/commands/source_commands.py new file mode 100644 index 0000000000000000000000000000000000000000..0edac246ff492d17688d58bf3657a1ba2c0dd91b --- /dev/null +++ b/commands/source_commands.py @@ -0,0 +1,270 @@ +import time +from typing import Any, Dict, List, Optional + +from loguru import logger +from pydantic import BaseModel +from surreal_commands import CommandInput, CommandOutput, command + +from open_notebook.database.repository import ensure_record_id +from open_notebook.domain.notebook import Source +from open_notebook.domain.transformation import Transformation +from open_notebook.exceptions import ConfigurationError + +try: + from open_notebook.graphs.source import source_graph + from open_notebook.graphs.transformation import graph as transform_graph +except ImportError as e: + logger.error(f"Failed to import graphs: {e}") + raise ValueError("graphs not available") + + +def full_model_dump(model): + if isinstance(model, BaseModel): + return model.model_dump() + elif isinstance(model, dict): + return {k: full_model_dump(v) for k, v in model.items()} + elif isinstance(model, list): + return [full_model_dump(item) for item in model] + else: + return model + + +class SourceProcessingInput(CommandInput): + source_id: str + content_state: Dict[str, Any] + notebook_ids: List[str] + transformations: List[str] + embed: bool + + +class SourceProcessingOutput(CommandOutput): + success: bool + source_id: str + embedded_chunks: int = 0 + insights_created: int = 0 + processing_time: float + error_message: Optional[str] = None + + +@command( + "process_source", + app="open_notebook", + retry={ + "max_attempts": 15, # Handle deep queues (workaround for SurrealDB v2 transaction conflicts) + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 120, # Allow queue to drain + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", # Avoid log noise during transaction conflicts + }, +) +async def process_source_command( + input_data: SourceProcessingInput, +) -> SourceProcessingOutput: + """ + Process source content using the source_graph workflow + """ + start_time = time.time() + + try: + logger.info(f"Starting source processing for source: {input_data.source_id}") + logger.info(f"Notebook IDs: {input_data.notebook_ids}") + logger.info(f"Transformations: {input_data.transformations}") + logger.info(f"Embed: {input_data.embed}") + + # 1. Load transformation objects from IDs + transformations = [] + for trans_id in input_data.transformations: + logger.info(f"Loading transformation: {trans_id}") + transformation = await Transformation.get(trans_id) + if not transformation: + raise ValueError(f"Transformation '{trans_id}' not found") + transformations.append(transformation) + + logger.info(f"Loaded {len(transformations)} transformations") + + # 2. Get existing source record to update its command field + source = await Source.get(input_data.source_id) + if not source: + raise ValueError(f"Source '{input_data.source_id}' not found") + + # Update source with command reference + source.command = ( + ensure_record_id(input_data.execution_context.command_id) + if input_data.execution_context + else None + ) + await source.save() + + logger.info(f"Updated source {source.id} with command reference") + + # 3. Process source with all notebooks + logger.info(f"Processing source with {len(input_data.notebook_ids)} notebooks") + + # Execute source_graph with all notebooks + result = await source_graph.ainvoke( + { # type: ignore[arg-type] + "content_state": input_data.content_state, + "notebook_ids": input_data.notebook_ids, # Use notebook_ids (plural) as expected by SourceState + "apply_transformations": transformations, + "embed": input_data.embed, + "source_id": input_data.source_id, # Add the source_id to the state + } + ) + + processed_source = result["source"] + + # 4. Gather processing results (notebook associations handled by source_graph) + # Note: embedding is fire-and-forget (async job), so we can't query the + # count here — it hasn't completed yet. The embed_source_command logs + # the actual count when it finishes. + insights_list = await processed_source.get_insights() + insights_created = len(insights_list) + + processing_time = time.time() - start_time + embed_status = "submitted" if input_data.embed else "skipped" + logger.info( + f"Successfully processed source: {processed_source.id} in {processing_time:.2f}s" + ) + logger.info( + f"Created {insights_created} insights, embedding {embed_status}" + ) + + return SourceProcessingOutput( + success=True, + source_id=str(processed_source.id), + embedded_chunks=0, + insights_created=insights_created, + processing_time=processing_time, + ) + + except ValueError as e: + # Validation errors are permanent failures - don't retry + processing_time = time.time() - start_time + logger.error(f"Source processing failed: {e}") + return SourceProcessingOutput( + success=False, + source_id=input_data.source_id, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + logger.debug( + f"Transient error processing source {input_data.source_id}: {e}" + ) + raise + + +# ============================================================================= +# RUN TRANSFORMATION COMMAND +# ============================================================================= + + +class RunTransformationInput(CommandInput): + """Input for running a transformation on an existing source.""" + + source_id: str + transformation_id: str + + +class RunTransformationOutput(CommandOutput): + """Output from transformation command.""" + + success: bool + source_id: str + transformation_id: str + processing_time: float + error_message: Optional[str] = None + + +@command( + "run_transformation", + app="open_notebook", + retry={ + "max_attempts": 5, + "wait_strategy": "exponential_jitter", + "wait_min": 1, + "wait_max": 60, + "stop_on": [ValueError, ConfigurationError], # Don't retry validation/config errors + "retry_log_level": "debug", + }, +) +async def run_transformation_command( + input_data: RunTransformationInput, +) -> RunTransformationOutput: + """ + Run a transformation on an existing source to generate an insight. + + This command runs the transformation graph which: + 1. Loads the source and transformation + 2. Calls the LLM to generate insight content + 3. Creates the insight via create_insight command (fire-and-forget) + + Use this command for UI-triggered insight generation to avoid blocking + the HTTP request while the LLM processes. + + Retry Strategy: + - Retries up to 5 times for transient failures (network, timeout, etc.) + - Uses exponential-jitter backoff (1-60s) + - Does NOT retry permanent failures (ValueError for validation errors) + """ + start_time = time.time() + + try: + logger.info( + f"Running transformation {input_data.transformation_id} " + f"on source {input_data.source_id}" + ) + + # Load source + source = await Source.get(input_data.source_id) + if not source: + raise ValueError(f"Source '{input_data.source_id}' not found") + + # Load transformation + transformation = await Transformation.get(input_data.transformation_id) + if not transformation: + raise ValueError( + f"Transformation '{input_data.transformation_id}' not found" + ) + + # Run transformation graph (includes LLM call + insight creation) + await transform_graph.ainvoke( + input=dict(source=source, transformation=transformation) + ) + + processing_time = time.time() - start_time + logger.info( + f"Successfully ran transformation {input_data.transformation_id} " + f"on source {input_data.source_id} in {processing_time:.2f}s" + ) + + return RunTransformationOutput( + success=True, + source_id=input_data.source_id, + transformation_id=input_data.transformation_id, + processing_time=processing_time, + ) + + except ValueError as e: + # Validation errors are permanent failures - don't retry + processing_time = time.time() - start_time + logger.error( + f"Failed to run transformation {input_data.transformation_id} " + f"on source {input_data.source_id}: {e}" + ) + return RunTransformationOutput( + success=False, + source_id=input_data.source_id, + transformation_id=input_data.transformation_id, + processing_time=processing_time, + error_message=str(e), + ) + except Exception as e: + # Transient failure - will be retried (surreal-commands logs final failure) + logger.debug( + f"Transient error running transformation {input_data.transformation_id} " + f"on source {input_data.source_id}: {e}" + ) + raise diff --git a/migrations/1.surrealql b/migrations/1.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..8ca52ca80bb4c5642c7d86b25a570960eea024f7 --- /dev/null +++ b/migrations/1.surrealql @@ -0,0 +1,178 @@ + +DEFINE TABLE IF NOT EXISTS source SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS + asset + ON TABLE source + FLEXIBLE TYPE option; + +DEFINE FIELD IF NOT EXISTS title ON TABLE source TYPE option; +DEFINE FIELD IF NOT EXISTS topics ON TABLE source TYPE option>; +DEFINE FIELD IF NOT EXISTS full_text ON TABLE source TYPE option; + +DEFINE FIELD IF NOT EXISTS created ON source DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON source DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS source_embedding SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS source ON TABLE source_embedding TYPE record; +DEFINE FIELD IF NOT EXISTS order ON TABLE source_embedding TYPE int; +DEFINE FIELD IF NOT EXISTS content ON TABLE source_embedding TYPE string; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_embedding TYPE array; + +DEFINE TABLE IF NOT EXISTS source_insight SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS source ON TABLE source_insight TYPE record; +DEFINE FIELD IF NOT EXISTS insight_type ON TABLE source_insight TYPE string; +DEFINE FIELD IF NOT EXISTS content ON TABLE source_insight TYPE string; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_insight TYPE array; + + +DEFINE EVENT IF NOT EXISTS source_delete ON TABLE source WHEN ($after == NONE) THEN { + delete source_embedding where source == $before.id; + delete source_insight where source == $before.id; +}; + +DEFINE TABLE IF NOT EXISTS note SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS title ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS summary ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS content ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE note TYPE array; + +DEFINE FIELD IF NOT EXISTS created ON note DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON note DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS notebook SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS name ON TABLE notebook TYPE option; +DEFINE FIELD IF NOT EXISTS description ON TABLE notebook TYPE option; +DEFINE FIELD IF NOT EXISTS archived ON TABLE notebook TYPE option DEFAULT False; + + +DEFINE FIELD IF NOT EXISTS created ON notebook DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON notebook DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS reference +TYPE RELATION +FROM source TO notebook; + +DEFINE TABLE IF NOT EXISTS artifact +TYPE RELATION +FROM note TO notebook; + +DEFINE TABLE IF NOT EXISTS podcast_config SCHEMALESS; + +-- entender o analyzer +DEFINE ANALYZER IF NOT EXISTS my_analyzer TOKENIZERS blank,class,camel,punct FILTERS snowball(english), lowercase; + +DEFINE INDEX IF NOT EXISTS idx_source_title ON TABLE source COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_full_text ON TABLE source COLUMNS full_text SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_embed_chunk ON TABLE source_embedding COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_insight ON TABLE source_insight COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_note ON TABLE note COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_note_title ON TABLE note COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results + group by item_id ORDER BY relevance DESC LIMIT $match_count); + + +}; + + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding LIMIT $match_count)} + ELSE { [] }; + + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight LIMIT $match_count)} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM note LIMIT $match_count)} + + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_insight_search); + + let $note_results = $note_content_search; + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results + group by item_id ORDER BY similarity DESC LIMIT $match_count); + + +}; + +IF array::len(select * from open_notebook:default_models) == 0 THEN + CREATE open_notebook:default_models SET + default_chat_model= "" +END; diff --git a/migrations/10.surrealql b/migrations/10.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..445618031c24bbce31e992b6938d47c8e3a9a685 --- /dev/null +++ b/migrations/10.surrealql @@ -0,0 +1,29 @@ +-- Migration 10: Quiz and Flashcard System +-- Using SCHEMALESS to match domain models flexibly + +-- Flashcard table with FSRS spaced repetition fields +DEFINE TABLE flashcard SCHEMALESS; +DEFINE FIELD created ON flashcard DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON flashcard DEFAULT time::now() VALUE time::now(); + +-- Quiz Question table +DEFINE TABLE quiz_question SCHEMALESS; +DEFINE FIELD created ON quiz_question DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON quiz_question DEFAULT time::now() VALUE time::now(); + +-- Quiz Session table +DEFINE TABLE quiz_session SCHEMALESS; +DEFINE FIELD created ON quiz_session DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON quiz_session DEFAULT time::now() VALUE time::now(); + +-- User Study Stats table for gamification +DEFINE TABLE user_study_stats SCHEMALESS; +DEFINE FIELD created ON user_study_stats DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON user_study_stats DEFAULT time::now() VALUE time::now(); + +-- Indexes for efficient queries +DEFINE INDEX IF NOT EXISTS flashcard_due_idx ON flashcard FIELDS due; +DEFINE INDEX IF NOT EXISTS flashcard_state_idx ON flashcard FIELDS state; +DEFINE INDEX IF NOT EXISTS flashcard_notebook_idx ON flashcard FIELDS notebook_id; +DEFINE INDEX IF NOT EXISTS quiz_session_notebook_idx ON quiz_session FIELDS notebook_id; +DEFINE INDEX IF NOT EXISTS quiz_question_session_idx ON quiz_question FIELDS quiz_session_id; diff --git a/migrations/10_down.surrealql b/migrations/10_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..18fc61a73ce74ab06b706a290f2bfd605acf5097 --- /dev/null +++ b/migrations/10_down.surrealql @@ -0,0 +1,5 @@ +-- Migration 10 Down: Remove Quiz and Flashcard tables + +REMOVE TABLE flashcard; +REMOVE TABLE quiz_question; +REMOVE TABLE quiz_session; diff --git a/migrations/11.surrealql b/migrations/11.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..5d601998e63abd498a0ef6122520ca80c3298f18 --- /dev/null +++ b/migrations/11.surrealql @@ -0,0 +1,75 @@ +-- Migration 11: Quiz and Flashcard System with FSRS Spaced Repetition + +-- Flashcard table with FSRS scheduling fields +DEFINE TABLE flashcard SCHEMAFULL; +DEFINE FIELD user_id ON flashcard TYPE option; +DEFINE FIELD source_id ON flashcard TYPE option; +DEFINE FIELD notebook_id ON flashcard TYPE option; +DEFINE FIELD front ON flashcard TYPE string; +DEFINE FIELD back ON flashcard TYPE string; +DEFINE FIELD tags ON flashcard TYPE option>; +-- FSRS scheduling fields +DEFINE FIELD stability ON flashcard TYPE float DEFAULT 0; +DEFINE FIELD difficulty ON flashcard TYPE float DEFAULT 0; +DEFINE FIELD elapsed_days ON flashcard TYPE int DEFAULT 0; +DEFINE FIELD scheduled_days ON flashcard TYPE int DEFAULT 0; +DEFINE FIELD reps ON flashcard TYPE int DEFAULT 0; +DEFINE FIELD lapses ON flashcard TYPE int DEFAULT 0; +DEFINE FIELD state ON flashcard TYPE int DEFAULT 0; +DEFINE FIELD due ON flashcard TYPE option; +DEFINE FIELD last_review ON flashcard TYPE option; +DEFINE FIELD created ON flashcard TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated ON flashcard TYPE datetime DEFAULT time::now(); +DEFINE INDEX flashcard_due_idx ON flashcard FIELDS due; +DEFINE INDEX flashcard_notebook_idx ON flashcard FIELDS notebook_id; +DEFINE INDEX flashcard_source_idx ON flashcard FIELDS source_id; + +-- Quiz session table +DEFINE TABLE quiz_session SCHEMAFULL; +DEFINE FIELD notebook_id ON quiz_session TYPE string; +DEFINE FIELD title ON quiz_session TYPE option; +DEFINE FIELD question_count ON quiz_session TYPE int DEFAULT 0; +DEFINE FIELD correct_count ON quiz_session TYPE int DEFAULT 0; +DEFINE FIELD score ON quiz_session TYPE option; +DEFINE FIELD difficulty ON quiz_session TYPE string DEFAULT "mixed"; +DEFINE FIELD status ON quiz_session TYPE string DEFAULT "in_progress"; +DEFINE FIELD started_at ON quiz_session TYPE datetime DEFAULT time::now(); +DEFINE FIELD completed_at ON quiz_session TYPE option; +DEFINE FIELD created ON quiz_session TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated ON quiz_session TYPE datetime DEFAULT time::now(); +DEFINE INDEX quiz_session_notebook_idx ON quiz_session FIELDS notebook_id; +DEFINE INDEX quiz_session_status_idx ON quiz_session FIELDS status; + +-- Quiz question table +DEFINE TABLE quiz_question SCHEMAFULL; +DEFINE FIELD quiz_session_id ON quiz_question TYPE string; +DEFINE FIELD source_id ON quiz_question TYPE option; +DEFINE FIELD question ON quiz_question TYPE string; +DEFINE FIELD question_type ON quiz_question TYPE string DEFAULT "mcq"; +DEFINE FIELD options ON quiz_question TYPE array; +DEFINE FIELD correct_index ON quiz_question TYPE int; +DEFINE FIELD explanation ON quiz_question TYPE string; +DEFINE FIELD difficulty ON quiz_question TYPE string DEFAULT "medium"; +DEFINE FIELD user_answer ON quiz_question TYPE option; +DEFINE FIELD is_correct ON quiz_question TYPE option; +DEFINE FIELD answered_at ON quiz_question TYPE option; +DEFINE FIELD time_spent_seconds ON quiz_question TYPE option; +DEFINE FIELD created ON quiz_question TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated ON quiz_question TYPE datetime DEFAULT time::now(); +DEFINE INDEX quiz_question_session_idx ON quiz_question FIELDS quiz_session_id; + +-- User study stats for gamification +DEFINE TABLE user_study_stats SCHEMAFULL; +DEFINE FIELD user_id ON user_study_stats TYPE string; +DEFINE FIELD current_streak ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD longest_streak ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD total_xp ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD level ON user_study_stats TYPE int DEFAULT 1; +DEFINE FIELD badges ON user_study_stats TYPE array DEFAULT []; +DEFINE FIELD last_study_date ON user_study_stats TYPE option; +DEFINE FIELD total_quizzes_completed ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD total_flashcards_reviewed ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD total_correct_answers ON user_study_stats TYPE int DEFAULT 0; +DEFINE FIELD created ON user_study_stats TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated ON user_study_stats TYPE datetime DEFAULT time::now(); +DEFINE INDEX user_study_stats_user_idx ON user_study_stats FIELDS user_id; diff --git a/migrations/11_down.surrealql b/migrations/11_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..264e25a80bdaef81525dfb2bd2e2305e3eb71952 --- /dev/null +++ b/migrations/11_down.surrealql @@ -0,0 +1,6 @@ +-- Migration 11 Down: Remove Quiz and Flashcard System + +REMOVE TABLE flashcard; +REMOVE TABLE quiz_session; +REMOVE TABLE quiz_question; +REMOVE TABLE user_study_stats; diff --git a/migrations/12.surrealql b/migrations/12.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..5b4f7bebd1f2f1568cbd5856090d4a4ebd24ade7 --- /dev/null +++ b/migrations/12.surrealql @@ -0,0 +1,49 @@ +-- Migration 12: Knowledge Graph Tables +-- Creates tables for concept nodes and edges + +-- Concept nodes extracted from sources +DEFINE TABLE IF NOT EXISTS concept_node SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS label ON concept_node TYPE string; +DEFINE FIELD IF NOT EXISTS type ON concept_node TYPE string DEFAULT 'concept'; -- concept, person, event, place, organization +DEFINE FIELD IF NOT EXISTS description ON concept_node TYPE option; +DEFINE FIELD IF NOT EXISTS source_ids ON concept_node TYPE array DEFAULT []; +DEFINE FIELD IF NOT EXISTS notebook_id ON concept_node TYPE option; +DEFINE FIELD IF NOT EXISTS embedding ON concept_node TYPE option; +DEFINE FIELD IF NOT EXISTS importance ON concept_node TYPE float DEFAULT 0.5; -- 0-1 scale +DEFINE FIELD IF NOT EXISTS mentions ON concept_node TYPE int DEFAULT 1; +DEFINE FIELD IF NOT EXISTS created ON concept_node TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON concept_node TYPE datetime DEFAULT time::now(); + +-- Indexes for concept nodes +DEFINE INDEX IF NOT EXISTS idx_concept_notebook ON concept_node FIELDS notebook_id; +DEFINE INDEX IF NOT EXISTS idx_concept_label ON concept_node FIELDS label; +DEFINE INDEX IF NOT EXISTS idx_concept_type ON concept_node FIELDS type; + +-- Concept edges (relationships between concepts) +DEFINE TABLE IF NOT EXISTS concept_edge SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS source_node ON concept_edge TYPE string; -- concept_node ID +DEFINE FIELD IF NOT EXISTS target_node ON concept_edge TYPE string; -- concept_node ID +DEFINE FIELD IF NOT EXISTS relationship ON concept_edge TYPE string; -- is_a, part_of, causes, related_to, etc. +DEFINE FIELD IF NOT EXISTS description ON concept_edge TYPE option; +DEFINE FIELD IF NOT EXISTS weight ON concept_edge TYPE float DEFAULT 1.0; +DEFINE FIELD IF NOT EXISTS source_ids ON concept_edge TYPE array DEFAULT []; +DEFINE FIELD IF NOT EXISTS notebook_id ON concept_edge TYPE option; +DEFINE FIELD IF NOT EXISTS created ON concept_edge TYPE datetime DEFAULT time::now(); + +-- Indexes for concept edges +DEFINE INDEX IF NOT EXISTS idx_edge_source ON concept_edge FIELDS source_node; +DEFINE INDEX IF NOT EXISTS idx_edge_target ON concept_edge FIELDS target_node; +DEFINE INDEX IF NOT EXISTS idx_edge_notebook ON concept_edge FIELDS notebook_id; + +-- Knowledge graph metadata (tracks when graphs were built) +DEFINE TABLE IF NOT EXISTS knowledge_graph_meta SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS notebook_id ON knowledge_graph_meta TYPE string; +DEFINE FIELD IF NOT EXISTS node_count ON knowledge_graph_meta TYPE int DEFAULT 0; +DEFINE FIELD IF NOT EXISTS edge_count ON knowledge_graph_meta TYPE int DEFAULT 0; +DEFINE FIELD IF NOT EXISTS last_built ON knowledge_graph_meta TYPE datetime; +DEFINE FIELD IF NOT EXISTS build_status ON knowledge_graph_meta TYPE string DEFAULT 'pending'; -- pending, building, completed, error +DEFINE FIELD IF NOT EXISTS error_message ON knowledge_graph_meta TYPE option; +DEFINE FIELD IF NOT EXISTS created ON knowledge_graph_meta TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON knowledge_graph_meta TYPE datetime DEFAULT time::now(); + +DEFINE INDEX IF NOT EXISTS idx_kg_meta_notebook ON knowledge_graph_meta FIELDS notebook_id UNIQUE; diff --git a/migrations/12_down.surrealql b/migrations/12_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..a005aec2a118775e4c88fc61137365ab05c6090e --- /dev/null +++ b/migrations/12_down.surrealql @@ -0,0 +1,5 @@ +-- Migration 12 Down: Remove Knowledge Graph Tables + +REMOVE TABLE IF EXISTS knowledge_graph_meta; +REMOVE TABLE IF EXISTS concept_edge; +REMOVE TABLE IF EXISTS concept_node; diff --git a/migrations/13.surrealql b/migrations/13.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..69447df3461ea0caa485492c0146e22ec3b70d5e --- /dev/null +++ b/migrations/13.surrealql @@ -0,0 +1,48 @@ +-- Migration 13: Auto-Update Agent - Source Monitoring & Notifications + +-- Table for tracking monitored sources +DEFINE TABLE source_monitor SCHEMAFULL; + +DEFINE FIELD source_id ON TABLE source_monitor TYPE record; +DEFINE FIELD enabled ON TABLE source_monitor TYPE bool DEFAULT true; +DEFINE FIELD check_frequency ON TABLE source_monitor TYPE string DEFAULT 'daily'; -- hourly, daily, weekly +DEFINE FIELD last_checked_at ON TABLE source_monitor TYPE option; +DEFINE FIELD last_content_hash ON TABLE source_monitor TYPE option; +DEFINE FIELD consecutive_failures ON TABLE source_monitor TYPE int DEFAULT 0; +DEFINE FIELD created_at ON TABLE source_monitor TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated_at ON TABLE source_monitor TYPE datetime DEFAULT time::now(); + +DEFINE INDEX source_monitor_source_idx ON TABLE source_monitor FIELDS source_id UNIQUE; +DEFINE INDEX source_monitor_enabled_idx ON TABLE source_monitor FIELDS enabled; + +-- Table for update notifications +DEFINE TABLE update_notification SCHEMAFULL; + +DEFINE FIELD source_id ON TABLE source_monitor TYPE record; +DEFINE FIELD source_title ON TABLE update_notification TYPE string; +DEFINE FIELD change_summary ON TABLE update_notification TYPE string; +DEFINE FIELD diff_highlights ON TABLE update_notification TYPE array; +DEFINE FIELD old_content_preview ON TABLE update_notification TYPE option; +DEFINE FIELD new_content_preview ON TABLE update_notification TYPE option; +DEFINE FIELD severity ON TABLE update_notification TYPE string DEFAULT 'info'; -- info, warning, critical +DEFINE FIELD is_read ON TABLE update_notification TYPE bool DEFAULT false; +DEFINE FIELD is_dismissed ON TABLE update_notification TYPE bool DEFAULT false; +DEFINE FIELD created_at ON TABLE update_notification TYPE datetime DEFAULT time::now(); + +DEFINE INDEX notification_source_idx ON TABLE update_notification FIELDS source_id; +DEFINE INDEX notification_read_idx ON TABLE update_notification FIELDS is_read; +DEFINE INDEX notification_dismissed_idx ON TABLE update_notification FIELDS is_dismissed; +DEFINE INDEX notification_created_idx ON TABLE update_notification FIELDS created_at; + +-- Table for monitoring job history +DEFINE TABLE monitor_job_run SCHEMAFULL; + +DEFINE FIELD started_at ON TABLE monitor_job_run TYPE datetime DEFAULT time::now(); +DEFINE FIELD completed_at ON TABLE monitor_job_run TYPE option; +DEFINE FIELD status ON TABLE monitor_job_run TYPE string DEFAULT 'running'; -- running, completed, failed +DEFINE FIELD sources_checked ON TABLE monitor_job_run TYPE int DEFAULT 0; +DEFINE FIELD updates_found ON TABLE monitor_job_run TYPE int DEFAULT 0; +DEFINE FIELD errors ON TABLE monitor_job_run TYPE array DEFAULT []; + +DEFINE INDEX job_run_status_idx ON TABLE monitor_job_run FIELDS status; +DEFINE INDEX job_run_started_idx ON TABLE monitor_job_run FIELDS started_at; diff --git a/migrations/13_down.surrealql b/migrations/13_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..960d13069f31327b5e776eb48881a5f816957484 --- /dev/null +++ b/migrations/13_down.surrealql @@ -0,0 +1,5 @@ +-- Migration 13 Down: Remove Auto-Update Agent tables + +REMOVE TABLE IF EXISTS monitor_job_run; +REMOVE TABLE IF EXISTS update_notification; +REMOVE TABLE IF EXISTS source_monitor; diff --git a/migrations/14.surrealql b/migrations/14.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..712931e5c8cf1323319042e1d8a0c46a7f9a0c55 --- /dev/null +++ b/migrations/14.surrealql @@ -0,0 +1,64 @@ +-- Migration 14: Study Plan Generator +-- AI-generated personalized study schedules + +-- Study plan table +DEFINE TABLE study_plan SCHEMAFULL; +DEFINE FIELD notebook_id ON study_plan TYPE string; +DEFINE FIELD title ON study_plan TYPE string; +DEFINE FIELD description ON study_plan TYPE option; +DEFINE FIELD deadline ON study_plan TYPE datetime; +DEFINE FIELD available_hours_per_day ON study_plan TYPE float DEFAULT 2.0; +DEFINE FIELD total_study_hours ON study_plan TYPE float; +DEFINE FIELD status ON study_plan TYPE string DEFAULT 'active'; -- active, completed, paused, cancelled +DEFINE FIELD progress_percentage ON study_plan TYPE float DEFAULT 0.0; +DEFINE FIELD created_at ON study_plan TYPE datetime DEFAULT time::now(); +DEFINE FIELD updated_at ON study_plan TYPE datetime DEFAULT time::now(); +DEFINE INDEX idx_study_plan_notebook ON study_plan FIELDS notebook_id; +DEFINE INDEX idx_study_plan_status ON study_plan FIELDS status; +DEFINE INDEX idx_study_plan_deadline ON study_plan FIELDS deadline; + +-- Study topic (extracted from sources) +DEFINE TABLE study_topic SCHEMAFULL; +DEFINE FIELD plan_id ON study_topic TYPE string; +DEFINE FIELD name ON study_topic TYPE string; +DEFINE FIELD description ON study_topic TYPE option; +DEFINE FIELD difficulty ON study_topic TYPE string DEFAULT 'medium'; -- easy, medium, hard +DEFINE FIELD estimated_hours ON study_topic TYPE float; +DEFINE FIELD priority ON study_topic TYPE int DEFAULT 5; -- 1-10, higher = more important +DEFINE FIELD source_ids ON study_topic TYPE array DEFAULT []; +DEFINE FIELD prerequisites ON study_topic TYPE array DEFAULT []; -- topic IDs +DEFINE FIELD status ON study_topic TYPE string DEFAULT 'not_started'; -- not_started, in_progress, completed, skipped +DEFINE FIELD mastery_level ON study_topic TYPE float DEFAULT 0.0; -- 0-100 +DEFINE FIELD created_at ON study_topic TYPE datetime DEFAULT time::now(); +DEFINE INDEX idx_study_topic_plan ON study_topic FIELDS plan_id; +DEFINE INDEX idx_study_topic_status ON study_topic FIELDS status; + +-- Study session (individual study blocks) +DEFINE TABLE study_session SCHEMAFULL; +DEFINE FIELD plan_id ON study_session TYPE string; +DEFINE FIELD topic_id ON study_session TYPE string; +DEFINE FIELD scheduled_date ON study_session TYPE datetime; +DEFINE FIELD scheduled_duration_minutes ON study_session TYPE int; +DEFINE FIELD actual_start ON study_session TYPE option; +DEFINE FIELD actual_end ON study_session TYPE option; +DEFINE FIELD session_type ON study_session TYPE string DEFAULT 'learn'; -- learn, review, practice, quiz +DEFINE FIELD status ON study_session TYPE string DEFAULT 'scheduled'; -- scheduled, in_progress, completed, skipped, rescheduled +DEFINE FIELD notes ON study_session TYPE option; +DEFINE FIELD rating ON study_session TYPE option; -- 1-5 self-assessment +DEFINE FIELD created_at ON study_session TYPE datetime DEFAULT time::now(); +DEFINE INDEX idx_study_session_plan ON study_session FIELDS plan_id; +DEFINE INDEX idx_study_session_topic ON study_session FIELDS topic_id; +DEFINE INDEX idx_study_session_date ON study_session FIELDS scheduled_date; +DEFINE INDEX idx_study_session_status ON study_session FIELDS status; + +-- Study plan adjustments (AI suggestions for plan changes) +DEFINE TABLE plan_adjustment SCHEMAFULL; +DEFINE FIELD plan_id ON plan_adjustment TYPE string; +DEFINE FIELD adjustment_type ON plan_adjustment TYPE string; -- reschedule, add_review, extend_deadline, increase_hours +DEFINE FIELD reason ON plan_adjustment TYPE string; +DEFINE FIELD original_value ON plan_adjustment TYPE option; +DEFINE FIELD suggested_value ON plan_adjustment TYPE option; +DEFINE FIELD status ON plan_adjustment TYPE string DEFAULT 'pending'; -- pending, accepted, rejected +DEFINE FIELD created_at ON plan_adjustment TYPE datetime DEFAULT time::now(); +DEFINE INDEX idx_plan_adjustment_plan ON plan_adjustment FIELDS plan_id; +DEFINE INDEX idx_plan_adjustment_status ON plan_adjustment FIELDS status; diff --git a/migrations/14_down.surrealql b/migrations/14_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..091887b55fcec828f5da6ebd9667cac22be8fab3 --- /dev/null +++ b/migrations/14_down.surrealql @@ -0,0 +1,6 @@ +-- Migration 14 Down: Remove Study Plan Generator tables + +REMOVE TABLE IF EXISTS plan_adjustment; +REMOVE TABLE IF EXISTS study_session; +REMOVE TABLE IF EXISTS study_topic; +REMOVE TABLE IF EXISTS study_plan; diff --git a/migrations/15.surrealql b/migrations/15.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..8a215daf755c23f5babf32892180e7ec065944e2 --- /dev/null +++ b/migrations/15.surrealql @@ -0,0 +1,30 @@ +-- Migration 15: Fix Quiz/Flashcard tables to use SCHEMALESS format +-- This fixes the schema mismatch between domain models and database tables + +-- Drop old schema-full tables +REMOVE TABLE IF EXISTS flashcard; +REMOVE TABLE IF EXISTS quiz_question; +REMOVE TABLE IF EXISTS quiz_session; +REMOVE TABLE IF EXISTS user_study_stats; + +-- Recreate as SCHEMALESS to match domain models +DEFINE TABLE flashcard SCHEMALESS; +DEFINE FIELD created ON flashcard DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON flashcard DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE quiz_question SCHEMALESS; +DEFINE FIELD created ON quiz_question DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON quiz_question DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE quiz_session SCHEMALESS; +DEFINE FIELD created ON quiz_session DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON quiz_session DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE user_study_stats SCHEMALESS; +DEFINE FIELD created ON user_study_stats DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON user_study_stats DEFAULT time::now() VALUE time::now(); + +-- Indexes +DEFINE INDEX IF NOT EXISTS flashcard_notebook_idx ON flashcard FIELDS notebook_id; +DEFINE INDEX IF NOT EXISTS quiz_session_notebook_idx ON quiz_session FIELDS notebook_id; +DEFINE INDEX IF NOT EXISTS quiz_question_session_idx ON quiz_question FIELDS quiz_session_id; diff --git a/migrations/15_down.surrealql b/migrations/15_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..84b580e81cbe67f1be1963014f5c8a7c9c61aee2 --- /dev/null +++ b/migrations/15_down.surrealql @@ -0,0 +1,2 @@ +-- Migration 15 Down: Reverse the table recreation (no-op for down since we can't restore old data) +-- Tables remain as SCHEMALESS diff --git a/migrations/16.surrealql b/migrations/16.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..9cc021d239682fc2859a16b92f472b4fad13c133 --- /dev/null +++ b/migrations/16.surrealql @@ -0,0 +1,79 @@ +-- Migration 16: Novel Feature Tables +-- Creates tables for knowledge graph, study plans, source updates, and research + +-- ===================================================== +-- Knowledge Graph Tables +-- ===================================================== + +-- Concept nodes in the knowledge graph +DEFINE TABLE concept_node SCHEMALESS; +DEFINE FIELD created ON concept_node DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON concept_node DEFAULT time::now() VALUE time::now(); +DEFINE INDEX concept_node_notebook_idx ON concept_node FIELDS notebook_id; +DEFINE INDEX concept_node_label_idx ON concept_node FIELDS label; + +-- Edges/relationships between concept nodes +DEFINE TABLE concept_edge SCHEMALESS; +DEFINE FIELD created ON concept_edge DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON concept_edge DEFAULT time::now() VALUE time::now(); +DEFINE INDEX concept_edge_notebook_idx ON concept_edge FIELDS notebook_id; +DEFINE INDEX concept_edge_source_idx ON concept_edge FIELDS source_node; +DEFINE INDEX concept_edge_target_idx ON concept_edge FIELDS target_node; + +-- Metadata about a notebook's knowledge graph +DEFINE TABLE knowledge_graph_meta SCHEMALESS; +DEFINE FIELD created ON knowledge_graph_meta DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON knowledge_graph_meta DEFAULT time::now() VALUE time::now(); +DEFINE INDEX knowledge_graph_meta_notebook_idx ON knowledge_graph_meta FIELDS notebook_id; + +-- ===================================================== +-- Study Plan Tables +-- ===================================================== + +-- Study plans +DEFINE TABLE study_plan SCHEMALESS; +DEFINE FIELD created_at ON study_plan DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated_at ON study_plan DEFAULT time::now() VALUE time::now(); +DEFINE INDEX study_plan_notebook_idx ON study_plan FIELDS notebook_id; + +-- Study topics within a plan +DEFINE TABLE study_topic SCHEMALESS; +DEFINE FIELD created_at ON study_topic DEFAULT time::now() VALUE $before OR time::now(); +DEFINE INDEX study_topic_plan_idx ON study_topic FIELDS plan_id; + +-- Study sessions +DEFINE TABLE study_session SCHEMALESS; +DEFINE FIELD created_at ON study_session DEFAULT time::now() VALUE $before OR time::now(); +DEFINE INDEX study_session_plan_idx ON study_session FIELDS plan_id; +DEFINE INDEX study_session_topic_idx ON study_session FIELDS topic_id; +DEFINE INDEX study_session_date_idx ON study_session FIELDS scheduled_date; + +-- Plan adjustments/suggestions +DEFINE TABLE plan_adjustment SCHEMALESS; +DEFINE FIELD created_at ON plan_adjustment DEFAULT time::now() VALUE $before OR time::now(); +DEFINE INDEX plan_adjustment_plan_idx ON plan_adjustment FIELDS plan_id; + +-- ===================================================== +-- Source Update Tables +-- ===================================================== + +-- Tracked source updates +DEFINE TABLE source_update SCHEMALESS; +DEFINE FIELD created ON source_update DEFAULT time::now() VALUE $before OR time::now(); +DEFINE INDEX source_update_source_idx ON source_update FIELDS source_id; +DEFINE INDEX source_update_status_idx ON source_update FIELDS status; + +-- Update job status +DEFINE TABLE update_job SCHEMALESS; +DEFINE FIELD created ON update_job DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON update_job DEFAULT time::now() VALUE time::now(); + +-- ===================================================== +-- Research Tables +-- ===================================================== + +-- Research sessions +DEFINE TABLE research_session SCHEMALESS; +DEFINE FIELD created ON research_session DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD updated ON research_session DEFAULT time::now() VALUE time::now(); +DEFINE INDEX research_session_notebook_idx ON research_session FIELDS notebook_id; diff --git a/migrations/16_down.surrealql b/migrations/16_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..2478e0d99a6172e3fc2ec6bf49c4583ba8a5199b --- /dev/null +++ b/migrations/16_down.surrealql @@ -0,0 +1,12 @@ +-- Migration 16 Down: Remove novel feature tables + +REMOVE TABLE IF EXISTS concept_node; +REMOVE TABLE IF EXISTS concept_edge; +REMOVE TABLE IF EXISTS knowledge_graph_meta; +REMOVE TABLE IF EXISTS study_plan; +REMOVE TABLE IF EXISTS study_topic; +REMOVE TABLE IF EXISTS study_session; +REMOVE TABLE IF EXISTS plan_adjustment; +REMOVE TABLE IF EXISTS source_update; +REMOVE TABLE IF EXISTS update_job; +REMOVE TABLE IF EXISTS research_session; diff --git a/migrations/17.surrealql b/migrations/17.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..d190b5122c668273a2ba75f0c4e174d9bda21184 --- /dev/null +++ b/migrations/17.surrealql @@ -0,0 +1,17 @@ +-- Migration 17: Fix source_monitor and update_notification source_id field type +-- Change from record to string to allow flexibility in source_id storage + +-- First, remove the old field definitions and recreate with string type +REMOVE FIELD IF EXISTS source_id ON TABLE source_monitor; +DEFINE FIELD source_id ON TABLE source_monitor TYPE string; + +-- Also fix update_notification table which had the same issue +REMOVE FIELD IF EXISTS source_id ON TABLE update_notification; +DEFINE FIELD source_id ON TABLE update_notification TYPE string; + +-- Recreate indexes +REMOVE INDEX IF EXISTS source_monitor_source_idx ON TABLE source_monitor; +DEFINE INDEX source_monitor_source_idx ON TABLE source_monitor FIELDS source_id UNIQUE; + +REMOVE INDEX IF EXISTS notification_source_idx ON TABLE update_notification; +DEFINE INDEX notification_source_idx ON TABLE update_notification FIELDS source_id; diff --git a/migrations/17_down.surrealql b/migrations/17_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..17746764000eee1b52173b69ed0b70e37e7e40a3 --- /dev/null +++ b/migrations/17_down.surrealql @@ -0,0 +1,8 @@ +-- Migration 17 down: Revert source_id field type changes +-- Note: This will cause data loss if there are existing records with string source_ids + +REMOVE FIELD IF EXISTS source_id ON TABLE source_monitor; +DEFINE FIELD source_id ON TABLE source_monitor TYPE record; + +REMOVE FIELD IF EXISTS source_id ON TABLE update_notification; +DEFINE FIELD source_id ON TABLE update_notification TYPE record; diff --git a/migrations/18.surrealql b/migrations/18.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..bedae2129bdca66f6fbab8015926e2644fea60e2 --- /dev/null +++ b/migrations/18.surrealql @@ -0,0 +1,12 @@ +-- Migration 18: Ensure FREE tier models are configured +-- This migration doesn't change podcast settings (user will configure later) +-- Just ensures the system is ready for Groq + Gemini usage + +-- Note: Podcast models will remain as originally configured +-- User will update podcast settings later when needed + +-- This is a placeholder migration to track version +-- Actual model selection happens in the UI via model defaults + +-- No-op query to satisfy migration runner (SurrealDB 2.x compatible) +INFO FOR DB; \ No newline at end of file diff --git a/migrations/18_down.surrealql b/migrations/18_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..9bfb8f8c6b41a0216e8a356f1809fcc21f1ca48c --- /dev/null +++ b/migrations/18_down.surrealql @@ -0,0 +1,2 @@ +-- Migration 18 Down: No changes to revert +-- This was a placeholder migration diff --git a/migrations/1_down.surrealql b/migrations/1_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..e53791aac000cbe983a2846652bbfd06fdf58c72 --- /dev/null +++ b/migrations/1_down.surrealql @@ -0,0 +1,24 @@ +REMOVE TABLE IF EXISTS source; +REMOVE TABLE IF EXISTS source_embedding; +REMOVE TABLE IF EXISTS source_insight; +REMOVE TABLE IF EXISTS note; +REMOVE TABLE IF EXISTS notebook; +REMOVE TABLE IF EXISTS reference; +REMOVE TABLE IF EXISTS artifact; +REMOVE TABLE IF EXISTS podcast_config; + +REMOVE EVENT IF EXISTS source_delete ON TABLE source; + +REMOVE ANALYZER IF EXISTS my_analyzer; + +REMOVE INDEX IF EXISTS idx_source_title ON TABLE source; +REMOVE INDEX IF EXISTS idx_source_full_text ON TABLE source; +REMOVE INDEX IF EXISTS idx_source_embed_chunk ON TABLE source_embedding; +REMOVE INDEX IF EXISTS idx_source_insight ON TABLE source_insight; +REMOVE INDEX IF EXISTS idx_note ON TABLE note; +REMOVE INDEX IF EXISTS idx_note_title ON TABLE note; + +REMOVE FUNCTION IF EXISTS fn::text_search; +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DELETE open_notebook:default_models; diff --git a/migrations/2.surrealql b/migrations/2.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..66c10172169fd49a942766c224032a5c19f7dd46 --- /dev/null +++ b/migrations/2.surrealql @@ -0,0 +1 @@ +DEFINE FIELD IF NOT EXISTS note_type ON TABLE note TYPE option; diff --git a/migrations/2_down.surrealql b/migrations/2_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..d0213a0e285d106b1f62afe0a5bade9c68db7752 --- /dev/null +++ b/migrations/2_down.surrealql @@ -0,0 +1 @@ +REMOVE FIELD IF EXISTS note_type ON TABLE note; diff --git a/migrations/3.surrealql b/migrations/3.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..f2a067f84171f9de24a310ed66fe78d0ca34d44e --- /dev/null +++ b/migrations/3.surrealql @@ -0,0 +1,145 @@ + +DEFINE TABLE IF NOT EXISTS chat_session SCHEMALESS; + +DEFINE TABLE IF NOT EXISTS refers_to +TYPE RELATION +FROM chat_session TO notebook; + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN ( + SELECT + id, title, content, parent_id, + math::max(similarity) as similarity + FROM $all_results + GROUP BY id + ORDER BY similarity DESC + LIMIT $match_count + ); +}; + + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results + where id is not None +group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/migrations/3_down.surrealql b/migrations/3_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..b8438e0a9448050b6de11821167f33e6c48eb3db --- /dev/null +++ b/migrations/3_down.surrealql @@ -0,0 +1,110 @@ +REMOVE TABLE IF EXISTS chat_session; + +REMOVE TABLE IF EXISTS refers_to; + + +REMOVE FUNCTION fn::vector_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding LIMIT $match_count)} + ELSE { [] }; + + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight LIMIT $match_count)} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM note LIMIT $match_count)} + + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_insight_search); + + let $note_results = $note_content_search; + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results + group by item_id ORDER BY similarity DESC LIMIT $match_count); + + +}; + +REMOVE FUNCTION fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results + group by item_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/migrations/4.surrealql b/migrations/4.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..77440870f8f61fb3add224c77c9fd94c587472ca --- /dev/null +++ b/migrations/4.surrealql @@ -0,0 +1,134 @@ + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + (source.title OR '') as title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (select id, parent_id, title, math::max(relevance) as relevance + from $final_results where id is not None + group by id, parent_id, title ORDER BY relevance DESC LIMIT $match_count); + +}; + + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/migrations/4_down.surrealql b/migrations/4_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..67acbd7166cc985bd68d4d6dbd6027550f93bbb2 --- /dev/null +++ b/migrations/4_down.surrealql @@ -0,0 +1,139 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN ( + SELECT + id, title, content, parent_id, + math::max(similarity) as similarity + FROM $all_results + GROUP BY id + ORDER BY similarity DESC + LIMIT $match_count + ); +}; + + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results + where id is not None +group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/migrations/5.surrealql b/migrations/5.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..3fd2a51522e7c5ea187ba12b40ac06268ec30105 --- /dev/null +++ b/migrations/5.surrealql @@ -0,0 +1,169 @@ + +-- remove old transformation defaults + +DELETE open_notebook:default_transformations; + +-- set up the default transformations + +DEFINE TABLE IF NOT EXISTS transformation SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS name ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS title ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS prompt ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS apply_default ON TABLE transformation TYPE bool DEFAULT False; +DEFINE FIELD IF NOT EXISTS created ON transformation DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON transformation DEFAULT time::now() VALUE time::now(); + + +insert into transformation [ + { + name: "Analyze Paper", + title: "Paper Analysis", + description: "Analyses a technical/scientific paper", + prompt:"# IDENTITY and PURPOSE + +You are an insightful and analytical reader of academic papers, extracting the key components, significance, and broader implications. Your focus is to uncover the core contributions, practical applications, methodological strengths or weaknesses, and any surprising findings. You are especially attuned to the clarity of arguments, the relevance to existing literature, and potential impacts on both the specific field and broader contexts. + +# STEPS + +1. **READ AND UNDERSTAND THE PAPER**: Thoroughly read the paper, identifying its main focus, arguments, methods, results, and conclusions. + +2. **IDENTIFY CORE ELEMENTS**: + - **Purpose**: What is the main goal or research question? + - **Contribution**: What new knowledge or innovation does this paper bring to the field? + - **Methods**: What methods are used, and are they novel or particularly effective? + - **Key Findings**: What are the most critical results, and why do they matter? + - **Limitations**: Are there any notable limitations or areas for further research? + +3. **SYNTHESIZE THE MAIN POINTS**: + - Extract the key elements and organize them into insightful observations. + - Highlight the broader impact and potential applications. + - Note any aspects that challenge established views or introduce new questions. + +# OUTPUT INSTRUCTIONS + +- Structure the output as follows: + - **PURPOSE**: A concise summary of the main research question or goal (1-2 sentences). + - **CONTRIBUTION**: A bullet list of 2-3 points that describe what the paper adds to the field. + - **KEY FINDINGS**: A bullet list of 2-3 points summarizing the critical outcomes of the study. + - **IMPLICATIONS**: A bullet list of 2-3 points discussing the significance or potential impact of the findings on the field or broader context. + - **LIMITATIONS**: A bullet list of 1-2 points identifying notable limitations or areas for future work. + +- **Bullet Points** should be between 15-20 words. +- Avoid starting each bullet point with the same word to maintain variety. +- Use clear and concise language that conveys the key ideas effectively. +- Do not include warnings, disclaimers, or personal opinions. +- Output only the requested sections with their respective labels.", + apply_default: False + }, + { + name: "Key Insights", + title: "Key Insights", + description: "Extracts important insights and actionable items", + prompt:"# IDENTITY and PURPOSE + +You extract surprising, powerful, and interesting insights from text content. You are interested in insights related to the purpose and meaning of life, human flourishing, the role of technology in the future of humanity, artificial intelligence and its affect on humans, memes, learning, reading, books, continuous improvement, and similar topics. +You create 15 word bullet points that capture the most important insights from the input. +Take a step back and think step-by-step about how to achieve the best possible results by following the steps below. + +# STEPS + +- Extract 20 to 50 of the most surprising, insightful, and/or interesting ideas from the input in a section called IDEAS, and write them on a virtual whiteboard in your mind using 15 word bullets. If there are less than 50 then collect all of them. Make sure you extract at least 20. + +- From those IDEAS, extract the most powerful and insightful of them and write them in a section called INSIGHTS. Make sure you extract at least 10 and up to 25. + +# OUTPUT INSTRUCTIONS + +- INSIGHTS are essentially higher-level IDEAS that are more abstracted and wise. +- Output the INSIGHTS section only. +- Each bullet should be about 15 words in length. +- Do not give warnings or notes; only output the requested sections. +- You use bulleted lists for output, not numbered lists. +- Do not start items with the same opening words. +- Ensure you follow ALL these instructions when creating your output. +", + apply_default: False + }, +{ + name: "Dense Summary", + title: "Dense Summary", + description: "Creates a rich, deep summary of the content", + prompt:"# MISSION +You are a Sparse Priming Representation (SPR) writer. An SPR is a particular kind of use of language for advanced NLP, NLU, and NLG tasks, particularly useful for the latest generation of Large Language Models (LLMs). You will be given information by the USER which you are to render as an SPR. + +# THEORY +LLMs are a kind of deep neural network. They have been demonstrated to embed knowledge, abilities, and concepts, ranging from reasoning to planning, and even to theory of mind. These are called latent abilities and latent content, collectively referred to as latent space. The latent space of an LLM can be activated with the correct series of words as inputs, which will create a useful internal state of the neural network. This is not unlike how the right shorthand cues can prime a human mind to think in a certain way. Like human minds, LLMs are associative, meaning you only need to use the correct associations to 'prime' another model to think in the same way. + +# METHODOLOGY +Render the input as a distilled list of succinct statements, assertions, associations, concepts, analogies, and metaphors. The idea is to capture as much, conceptually, as possible but with as few words as possible. Write it in a way that makes sense to you, as the future audience will be another language model, not a human. Use complete sentences. +", + apply_default: True + }, +{ + name: "Reflections", + title: "Reflection Questions", + description: "Generates reflection questions from the document to help explore it further", + prompt:"# IDENTITY and PURPOSE + +You extract deep, thought-provoking, and meaningful reflections from text content. You are especially focused on themes related to the human experience, such as the purpose of life, personal growth, the intersection of technology and humanity, artificial intelligence's societal impact, human potential, collective evolution, and transformative learning. Your reflections aim to provoke new ways of thinking, challenge assumptions, and provide a thoughtful synthesis of the content. + +# STEPS + +- Extract 3 to 5 of the most profound, thought-provoking, and/or meaningful ideas from the input in a section called REFLECTIONS. +- Each reflection should aim to explore underlying implications, connections to broader human experiences, or highlight a transformative perspective. +- Take a step back and consider the deeper significance or questions that arise from the content. + +# OUTPUT INSTRUCTIONS + +- The output section should be labeled as REFLECTIONS. +- Each bullet point should be between 20-25 words. +- Avoid repetition in the phrasing and ensure variety in sentence structure. +- The reflections should encourage deeper inquiry and provide a synthesis that transcends surface-level observations. +- Use bullet points, not numbered lists. +- Every bullet should be formatted as a question that elicits contemplation or a statement that offers a profound insight. +- Do not give warnings or notes; only output the requested section.", + apply_default: False + }, +{ + name: "Table of Contents", + title: "Table of Contents", + description: "Describes the different topics of the document", + prompt:"# SYSTEM ROLE +You are a content analysis assistant that reads through documents and provides a Table of Contents (ToC) to help users identify what the document covers more easily. +Your ToC should capture all major topics and transitions in the content and should mention them in the order theh appear. + +# TASK +Analyze the provided content and create a Table of Contents: +- Captures the core topics included in the text +- Gives a small description of what is covered", + apply_default: False + }, +{ + name: "Simple Summary", + title: "Simple Summary", + description: "Generates a small summary of the content", + prompt:"# SYSTEM ROLE +You are a content summarization assistant that creates dense, information-rich summaries optimized for machine understanding. Your summaries should capture key concepts with minimal words while maintaining complete, clear sentences. + +# TASK +Analyze the provided content and create a summary that: +- Captures the core concepts and key information +- Uses clear, direct language +- Maintains context from any previous summaries", + apply_default: False + }, +]; + +-- Sets the default transformation instructions prompt +UPSERT open_notebook:default_prompts + CONTENT {transformation_instructions: "# INSTRUCTIONS + + You are my learning assistant and you help me process and transform content so that I can extract insights from them. + + # IMPORTANT + - You are working on my editorial projects. The text below is my own. Do not give me any warnings about copyright or plagiarism. + - Output ONLY the requested content, without acknowledgements of the task and additional chatting. Don't start with \"Sure, I can help you with that.\" or \"Here is the information you requested:\". Just provide the content. + - Do not stop in the middle of the generation to ask me questions. Execute my request completely. + "}; + diff --git a/migrations/5_down.surrealql b/migrations/5_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..a9e3eb2c6bcacfc7a7bb20de1b6364798359f186 --- /dev/null +++ b/migrations/5_down.surrealql @@ -0,0 +1,2 @@ + +REMOVE TABLE IF EXISTS transformation SCHEMAFULL; diff --git a/migrations/6.surrealql b/migrations/6.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..77c188b70cc9536fa6cbd5c6451fa1b004e69689 --- /dev/null +++ b/migrations/6.surrealql @@ -0,0 +1 @@ +update model set provider='vertex' where provider='vertexai'; \ No newline at end of file diff --git a/migrations/6_down.surrealql b/migrations/6_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..8a7e09d318ce9e752cb436dc1535e0c05d9454a9 --- /dev/null +++ b/migrations/6_down.surrealql @@ -0,0 +1 @@ +update model set provider='vertexai' where provider='vertex'; \ No newline at end of file diff --git a/migrations/7.surrealql b/migrations/7.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..064ae2a1283baf8c80b65364a42901e1ef16c35c --- /dev/null +++ b/migrations/7.surrealql @@ -0,0 +1,152 @@ +DEFINE TABLE IF NOT EXISTS episode_profile SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE episode_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speaker_config ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS outline_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS outline_model ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS transcript_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS transcript_model ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS default_briefing ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS num_segments ON TABLE episode_profile TYPE int DEFAULT 5; +DEFINE FIELD IF NOT EXISTS created ON TABLE episode_profile TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON TABLE episode_profile TYPE datetime DEFAULT time::now(); + +-- Create Speaker Profile table +remove table speaker_profile; +DEFINE TABLE IF NOT EXISTS speaker_profile SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS tts_provider ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS tts_model ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS speakers ON TABLE speaker_profile TYPE array; +DEFINE FIELD IF NOT EXISTS speakers.*.name ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS speakers.*.voice_id ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speakers.*.backstory ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speakers.*.personality ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS created ON TABLE speaker_profile TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON TABLE speaker_profile TYPE datetime DEFAULT time::now(); + + +-- Enhance PodcastEpisode table +DEFINE TABLE IF NOT EXISTS episode SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS created ON episode DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON episode DEFAULT time::now() VALUE time::now(); +DEFINE FIELD IF NOT EXISTS name ON TABLE episode TYPE string; +DEFINE FIELD IF NOT EXISTS briefing ON TABLE episode TYPE option; +DEFINE FIELD IF NOT EXISTS episode_profile ON TABLE episode FLEXIBLE TYPE object; +DEFINE FIELD IF NOT EXISTS speaker_profile ON TABLE episode FLEXIBLE TYPE object; +DEFINE FIELD IF NOT EXISTS transcript ON TABLE episode FLEXIBLE TYPE option; +DEFINE FIELD IF NOT EXISTS outline ON TABLE episode FLEXIBLE TYPE option; +DEFINE FIELD IF NOT EXISTS command ON TABLE episode TYPE option>; +DEFINE FIELD IF NOT EXISTS content ON TABLE episode TYPE option; +DEFINE FIELD IF NOT EXISTS audio_file ON TABLE episode TYPE option; + +-- Create indexes for better performance +DEFINE INDEX IF NOT EXISTS idx_episode_profile_name ON TABLE episode_profile COLUMNS name UNIQUE CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_speaker_profile_name ON TABLE speaker_profile COLUMNS name UNIQUE CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_episode_profile ON TABLE episode COLUMNS episode_profile CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_episode_command ON TABLE episode COLUMNS command CONCURRENTLY; + + +--Sample data + +insert into episode_profile +[ + { + name: "tech_discussion", + description: "Technical discussion between 2 experts", + speaker_config: "tech_experts", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Create an engaging technical discussion about the provided content. Focus on practical insights, real-world applications, and detailed explanations that would interest developers and technical professionals.", + num_segments: 5 + }, + { + name: "solo_expert", + description: "Single expert explaining complex topics", + speaker_config: "solo_expert", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Create an educational explanation of the provided content. Break down complex concepts into digestible segments, use analogies and examples, and maintain an engaging teaching style.", + "num_segments":4 }, + { + name: "business_analysis", + description: "Business-focused analysis and discussion", + speaker_config: "business_panel", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Analyze the provided content from a business perspective. Discuss market implications, strategic insights, competitive advantages, and actionable business intelligence.", + "num_segments":6 } + ]; + +insert into speaker_profile +[ + { + name: "tech_experts", + description: "Two technical experts for tech discussions", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Dr. Alex Chen", + voice_id: "nova", + backstory: "Senior AI researcher and former tech lead at major companies. Specializes in making complex technical concepts accessible.", + personality: "Analytical, clear communicator, asks probing questions to dig deeper into technical details" + }, + { + name: "Jamie Rodriguez", + voice_id: "alloy", + backstory: "Full-stack engineer and tech entrepreneur. Loves practical applications and real-world implementations.", + personality: "Enthusiastic, practical-minded, great at explaining implementation details and trade-offs" + } + ] + }, + { + name: "solo_expert", + description: "Single expert for educational content", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Professor Sarah Kim", + voice_id: "nova", + backstory: "Distinguished professor and researcher. Has a gift for making complex topics accessible to broad audiences.", + personality: "Patient teacher, uses analogies and examples, breaks down complex concepts step by step" + } + ] + }, + { + name: "business_panel", + description: "Business analysis panel with diverse perspectives", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Marcus Thompson", + voice_id: "echo", + backstory: "Former McKinsey consultant, now startup advisor. Expert in strategic analysis and market dynamics.", + personality: "Strategic thinker, data-driven, excellent at identifying key insights and implications" + }, + { + name: "Elena Vasquez", + voice_id: "shimmer", + backstory: "Serial entrepreneur and investor. Focuses on practical implementation and execution.", + personality: "Action-oriented, pragmatic, brings startup experience and execution focus" + }, + { + name: "Johny Bing", + voice_id: "ash", + backstory: "Youtube celebrity and business mogul. Focuses on practical implementation and execution.", + personality: "Controversial, likes to question ideas and concepts. He brings a fresh perspective and always has a point to make." + } + ] + } + ]; + + diff --git a/migrations/7_down.surrealql b/migrations/7_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..de41849ed0b5d9016925823f7ae0fb72bcc89ae6 --- /dev/null +++ b/migrations/7_down.surrealql @@ -0,0 +1,3 @@ +REMOVE TABLE IF EXISTS episode_profile; +REMOVE TABLE IF EXISTS speaker_profile; +REMOVE TABLE IF EXISTS episode; diff --git a/migrations/8.surrealql b/migrations/8.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..f08a844d147fc017bbdd0b0eea3367c455e0c6b7 --- /dev/null +++ b/migrations/8.surrealql @@ -0,0 +1,12 @@ + +-- Migration 8: Support chat sessions for both notebooks and sources +-- This migration allows chat_session to refer to either a notebook or a source + +DEFINE TABLE OVERWRITE refers_to +TYPE RELATION +FROM chat_session TO notebook|source; + +-- Add model_override field to chat_session for per-session model selection +DEFINE FIELD model_override ON chat_session TYPE option; + +DEFINE FIELD command ON source TYPE option>; \ No newline at end of file diff --git a/migrations/8_down.surrealql b/migrations/8_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..6bb26ceb7811f6a0c331380a737f2a1d30906279 --- /dev/null +++ b/migrations/8_down.surrealql @@ -0,0 +1,11 @@ + +-- Rollback Migration 8: Revert to notebook-only chat sessions + +DEFINE TABLE OVERWRITE refers_to +TYPE RELATION +FROM chat_session TO notebook; + +-- Remove model_override field from chat_session +REMOVE FIELD model_override ON chat_session; + +REMOVE FIELD command ON source; diff --git a/migrations/9.surrealql b/migrations/9.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..582f41fe70f2b13231e3197c90625cef017ba802 --- /dev/null +++ b/migrations/9.surrealql @@ -0,0 +1,66 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/migrations/9_down.surrealql b/migrations/9_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..b383a524c2846273d349be14c99b276ff5c73e01 --- /dev/null +++ b/migrations/9_down.surrealql @@ -0,0 +1,63 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/open_notebook/CLAUDE.md b/open_notebook/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..e92860a2a84dbb8268c3b073dd267785e85b45e2 --- /dev/null +++ b/open_notebook/CLAUDE.md @@ -0,0 +1,225 @@ +# Open Notebook - Root CLAUDE.md + +This file provides architectural guidance for contributors working on Open Notebook at the project level. + +## Project Overview + +**Open Notebook** is an open-source, privacy-focused alternative to Google's Notebook LM. It's an AI-powered research assistant enabling users to upload multi-modal content (PDFs, audio, video, web pages), generate intelligent notes, search semantically, chat with AI models, and produce professional podcasts—all with complete control over data and choice of AI providers. + +**Key Values**: Privacy-first, multi-provider AI support, fully self-hosted option, open-source transparency. + +--- + +## Three-Tier Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ Frontend (React/Next.js) │ +│ frontend/ @ port 3000 │ +├─────────────────────────────────────────────────────────┤ +│ - Notebooks, sources, notes, chat, podcasts, search UI │ +│ - Zustand state management, TanStack Query (React Query)│ +│ - Shadcn/ui component library with Tailwind CSS │ +└────────────────────────┬────────────────────────────────┘ + │ HTTP REST +┌────────────────────────▼────────────────────────────────┐ +│ API (FastAPI) │ +│ api/ @ port 5055 │ +├─────────────────────────────────────────────────────────┤ +│ - REST endpoints for notebooks, sources, notes, chat │ +│ - LangGraph workflow orchestration │ +│ - Job queue for async operations (podcasts) │ +│ - Multi-provider AI provisioning via Esperanto │ +└────────────────────────┬────────────────────────────────┘ + │ SurrealQL +┌────────────────────────▼────────────────────────────────┐ +│ Database (SurrealDB) │ +│ Graph database @ port 8000 │ +├─────────────────────────────────────────────────────────┤ +│ - Records: Notebook, Source, Note, ChatSession, Credential│ +│ - Relationships: source-to-notebook, note-to-source │ +│ - Vector embeddings for semantic search │ +└─────────────────────────────────────────────────────────┘ +``` + +--- + +## Useful sources + +User documentation is at @docs/ + +## Tech Stack + +### Frontend (`frontend/`) +- **Framework**: Next.js 16 (React 19) +- **Language**: TypeScript +- **State Management**: Zustand +- **Data Fetching**: TanStack Query (React Query) +- **Styling**: Tailwind CSS + Shadcn/ui +- **Build Tool**: Webpack (via Next.js) +- **i18n compatible**: All front-end changes must also consider the translation keys + +### API Backend (`api/` + `open_notebook/`) +- **Framework**: FastAPI 0.104+ +- **Language**: Python 3.11+ +- **Workflows**: LangGraph state machines +- **Database**: SurrealDB async driver +- **AI Providers**: Esperanto library (8+ providers: OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI) +- **Job Queue**: Surreal-Commands for async jobs (podcasts) +- **Logging**: Loguru +- **Validation**: Pydantic v2 +- **Testing**: Pytest + +### Database +- **SurrealDB**: Graph database with built-in embedding storage and vector search +- **Schema Migrations**: Automatic on API startup via AsyncMigrationManager + +### Additional Services +- **Content Processing**: content-core library (file/URL extraction) +- **Prompts**: AI-Prompter with Jinja2 templating +- **Podcast Generation**: podcast-creator library +- **Embeddings**: Multi-provider via Esperanto + +--- + +## Architecture Highlights + +### 1. Async-First Design +- All database queries, graph invocations, and API calls are async (await) +- SurrealDB async driver with connection pooling +- FastAPI handles concurrent requests efficiently + +### 2. LangGraph Workflows +- **source.py**: Content ingestion (extract → embed → save) +- **chat.py**: Conversational agent with message history +- **ask.py**: Search + synthesis (retrieve relevant sources → LLM) +- **transformation.py**: Custom transformations on sources +- All use `provision_langchain_model()` for smart model selection + +### 3. Multi-Provider AI +- **Esperanto library**: Unified interface to 8+ AI providers +- **Credential system**: Individual encrypted credential records per provider; models link to credentials for direct config +- **ModelManager**: Factory pattern with fallback logic; uses credential config when available, env vars as fallback +- **Smart selection**: Detects large contexts, prefers long-context models +- **Override support**: Per-request model configuration + +### 4. Database Schema +- **Automatic migrations**: AsyncMigrationManager runs on API startup +- **SurrealDB graph model**: Records with relationships and embeddings +- **Vector search**: Built-in semantic search across all content +- **Transactions**: Repo functions handle ACID operations + +### 5. Error Handling +- **Custom exceptions** (`exceptions.py`): Hierarchy rooted at `OpenNotebookError` with typed subclasses (`AuthenticationError`, `ConfigurationError`, `RateLimitError`, `ExternalServiceError`, `NetworkError`, etc.) +- **Error classification** (`utils/error_classifier.py`): `classify_error()` maps raw LLM provider exceptions to typed exceptions with user-friendly messages via keyword matching +- **Global handlers**: FastAPI exception handlers in `api/main.py` convert typed exceptions to appropriate HTTP status codes (401, 422, 429, 502, etc.) + +### 6. Authentication +- **Current**: Simple password middleware (insecure, dev-only) +- **Production**: Replace with OAuth/JWT (see CONFIGURATION.md) + +--- + +## Important Quirks & Gotchas + +### API Startup +- **Migrations run automatically** on startup; check logs for errors +- **Must start API before UI**: UI depends on API for all data +- **SurrealDB must be running**: API fails without database connection + +### Frontend-Backend Communication +- **Base API URL**: Configured in `.env.local` (default: http://localhost:5055) +- **CORS enabled**: Configured in `api/main.py` (allow all origins in dev) +- **Rate limiting**: Not built-in; add at proxy layer for production + +### LangGraph Workflows +- **Blocking operations**: Chat/podcast workflows may take minutes; no timeout +- **State persistence**: Uses SQLite checkpoint storage in `/data/sqlite-db/` +- **Model fallback**: If primary model fails, falls back to cheaper/smaller model + +### Podcast Generation +- **Async job queue**: `podcast_service.py` submits jobs but doesn't wait +- **Track status**: Use `/commands/{command_id}` endpoint to poll status +- **Failure handling**: Failed jobs are marked as "failed" with error messages; retry via `POST /podcasts/episodes/{id}/retry` +- **No automatic retries**: Podcast jobs use `max_attempts: 1` to prevent duplicate episode records +- **TTS failures**: Fall back to silent audio if speech synthesis fails + +### Content Processing +- **File extraction**: Uses content-core library; supports 50+ file types +- **URL handling**: Extracts text + metadata from web pages +- **Large files**: Content processing is sync; may block API briefly + +--- + +## Component References + +See dedicated CLAUDE.md files for detailed guidance: + +- **[frontend/CLAUDE.md](../frontend/CLAUDE.md)**: React/Next.js architecture, state management, API integration +- **[api/CLAUDE.md](../api/CLAUDE.md)**: FastAPI structure, service pattern, endpoint development +- **[domain/CLAUDE.md](domain/CLAUDE.md)**: Data models, repository pattern, search functions +- **[ai/CLAUDE.md](ai/CLAUDE.md)**: ModelManager, AI provider integration, Esperanto usage +- **[graphs/CLAUDE.md](graphs/CLAUDE.md)**: LangGraph workflow design, state machines +- **[database/CLAUDE.md](database/CLAUDE.md)**: SurrealDB operations, migrations, async patterns + +--- + +## Documentation Map + +- **[README.md](../README.md)**: Project overview, features, quick start +- **[docs/index.md](../docs/index.md)**: Complete user & deployment documentation +- **[CONFIGURATION.md](../CONFIGURATION.md)**: Environment variables, model configuration +- **[CONTRIBUTING.md](../CONTRIBUTING.md)**: Contribution guidelines +- **[MAINTAINER_GUIDE.md](../MAINTAINER_GUIDE.md)**: Release & maintenance procedures + +--- + +## Testing Strategy + +- **Unit tests**: `tests/test_domain.py`, `test_models_api.py` +- **Graph tests**: `tests/test_graphs.py` (workflow integration) +- **Utils tests**: `tests/test_utils.py`, `tests/test_chunking.py`, `tests/test_embedding.py` +- **Run all**: `uv run pytest tests/` +- **Coverage**: Check with `pytest --cov` + +--- + +## Common Tasks + +### Add a New API Endpoint +1. Create router in `api/routers/feature.py` +2. Create service in `api/feature_service.py` +3. Define schemas in `api/models.py` +4. Register router in `api/main.py` +5. Test via http://localhost:5055/docs + +### Add a New LangGraph Workflow +1. Create `open_notebook/graphs/workflow_name.py` +2. Define StateDict and node functions +3. Build graph with `.add_node()` / `.add_edge()` +4. Invoke in service: `graph.ainvoke({"input": ...}, config={"..."})` +5. Test with sample data in `tests/` + +### Add Database Migration +1. Create `migrations/XXX_description.surql` +2. Write SurrealQL schema changes +3. Create `migrations/XXX_description_down.surql` (optional rollback) +4. API auto-detects on startup; migration runs if newer than recorded version + +### Deploy to Production +1. Review [CONFIGURATION.md](CONFIGURATION.md) for security settings +2. Use `make docker-release` for multi-platform image +3. Push to Docker Hub / GitHub Container Registry +4. Deploy `docker compose --profile multi up` +5. Verify migrations via API logs + +--- + +## Support & Community + +- **Documentation**: https://open-notebook.ai +- **Discord**: https://discord.gg/37XJPXfz2w +- **Issues**: https://github.com/lfnovo/open-notebook/issues +- **License**: MIT (see LICENSE) + + diff --git a/open_notebook/__init__.py b/open_notebook/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/open_notebook/ai/CLAUDE.md b/open_notebook/ai/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..5e8de94f7bca871648dd687575d91179d71084e0 --- /dev/null +++ b/open_notebook/ai/CLAUDE.md @@ -0,0 +1,331 @@ +# AI Module + +Model configuration, provisioning, and management for multi-provider AI integration via Esperanto. + +## Purpose + +Centralizes AI model lifecycle: database models for model metadata (provider, type), default model configuration, and factory for instantiating LLM/embedding/speech models at runtime with fallback logic. + +## Architecture Overview + +**Two-tier system**: +1. **Database models** (`Model`, `DefaultModels`): Metadata storage and default configuration +2. **ModelManager**: Factory for provisioning models with intelligent fallback (large context detection, config override) + +All models use Esperanto library as provider abstraction (OpenAI, Anthropic, Google, Groq, Ollama, Mistral, DeepSeek, xAI, OpenRouter). + +## Component Catalog + +### models.py + +#### Model (ObjectModel) +- Database record: name, provider, type (language/embedding/speech_to_text/text_to_speech), credential (optional link to Credential record) +- `get_models_by_type()`: Async query to fetch all models of a specific type +- `get_credential_obj()`: Fetches linked Credential object (if credential field set) +- `get_by_credential(credential_id)`: Class method to find all models linked to a credential +- Stores provider-model pairs for AI factory instantiation + +#### DefaultModels (RecordModel) +- Singleton configuration record (record_id: `open_notebook:default_models`) +- Fields: default_chat_model, default_transformation_model, large_context_model, default_text_to_speech_model, default_speech_to_text_model, default_embedding_model, default_tools_model +- `get_instance()`: Always fetches fresh from database (overrides parent caching for real-time updates) +- Returns fresh instance on each call (no singleton cache) + +#### ModelManager +- Stateless factory for instantiating AI models +- `get_model(model_id)`: Retrieves Model by ID; if model has linked credential, uses `credential.to_esperanto_config()` for provider config; otherwise falls back to env var provisioning via `key_provider` +- `get_defaults()`: Fetches DefaultModels configuration +- `get_default_model(model_type)`: Smart lookup (e.g., "chat" → default_chat_model, "transformation" → default_transformation_model with fallback to chat) +- `get_speech_to_text()`, `get_text_to_speech()`, `get_embedding_model()`: Type-specific convenience methods with assertions +- **Global instance**: `model_manager` singleton exported for use throughout app + +### provision.py + +#### provision_langchain_model() +- Factory for LangGraph nodes needing LLM provisioning +- **Smart fallback logic**: + - If tokens > 105,000: Use `large_context_model` + - Elif `model_id` specified: Use specific model + - Else: Use default model for type (e.g., "chat", "transformation") +- Returns LangChain-compatible model via `.to_langchain()` +- Logs model selection decision + +### key_provider.py + +#### API Key Provider (Credential→Env Fallback) +- **Purpose**: Provides API keys from database first, falls back to environment variables +- **Pattern**: Before Esperanto creates a model, keys are loaded from `Credential` records and set as environment variables +- **Integration point**: Called by `ModelManager.get_model()` as fallback when model has no linked credential + +#### Key Functions +- `get_api_key(provider)`: Get single API key (DB first, then env var) +- `provision_provider_keys(provider)`: Set env vars from DB config for a provider +- `provision_all_keys()`: Load all provider keys from DB into env vars (useful at startup) + +#### Provider Configuration Maps +- `PROVIDER_CONFIG`: Simple providers (openai, anthropic, google, groq, etc.) +- `VERTEX_CONFIG`: Google Vertex AI (project, location, credentials) +- `AZURE_CONFIG`: Azure OpenAI (api_key, endpoint, api_version, mode-specific endpoints) +- `OPENAI_COMPATIBLE_CONFIG`: Generic OpenAI-compatible (generic + mode-specific for LLM/EMBEDDING/STT/TTS) + +## Common Patterns + +- **Type dispatch**: Model.type field drives factory logic (4 model types) +- **Provider abstraction**: Esperanto handles provider differences; ModelManager unaware of provider specifics +- **Fresh defaults**: DefaultModels.get_instance() always fetches from database (not cached) for live config updates +- **Config override**: provision_langchain_model() accepts kwargs passed to AIFactory.create_* methods +- **Token-based selection**: provision_langchain_model() detects large contexts and upgrades model automatically +- **Type assertions**: get_speech_to_text(), get_embedding_model() assert returned type (safety check) +- **Credential→Env fallback**: If model has linked credential, config from `credential.to_esperanto_config()` is used directly; otherwise keys checked in database via key_provider, then environment variables; enables UI-based key management while maintaining backward compatibility + +## Key Dependencies + +- `esperanto`: AIFactory.create_language(), create_embedding(), create_speech_to_text(), create_text_to_speech() +- `open_notebook.database.repository`: repo_query, ensure_record_id +- `open_notebook.domain.base`: ObjectModel, RecordModel base classes +- `open_notebook.domain.credential`: Credential for database-stored API keys +- `open_notebook.utils`: token_count() for context size detection +- `loguru`: Logging for model selection decisions + +## Important Quirks & Gotchas + +- **Token counting rough estimate**: provision_langchain_model() uses token_count() which estimates via cl100k_base encoding (may differ 5-10% from actual model) +- **Large context threshold hard-coded**: 105,000 token threshold for large_context_model upgrade (not configurable) +- **DefaultModels.get_instance() fresh fetch**: Intentionally bypasses parent singleton cache to pick up live config changes; creates new instance each call +- **Type-specific getters use assertions**: get_speech_to_text() asserts isinstance (catches misconfiguration early) +- **ConfigurationError on missing model**: ModelManager.get_model() and provision_langchain_model() raise `ConfigurationError` (not ValueError) when a model is not found or not configured, so the global exception handler returns HTTP 422 with a descriptive message +- **Esperanto caching**: Actual model instances cached by Esperanto (not by ModelManager); ModelManager stateless +- **Fallback chain specificity**: "transformation" type falls back to default_chat_model if not explicitly set (convention-based) +- **kwargs passed through**: provision_langchain_model() passes kwargs to AIFactory but doesn't validate what's accepted +- **Key provider sets env vars**: `provision_provider_keys()` modifies `os.environ` to inject DB-stored keys (from `Credential` records); Esperanto reads from env vars (only used as fallback when model has no linked credential) + +## How to Extend + +1. **Add new model type**: Add type string to Model.type enum, add create_* method in AIFactory, handle in ModelManager.get_model() +2. **Add new default configuration**: Extend DefaultModels with new field (e.g., default_vision_model), add getter in ModelManager +3. **Change fallback logic**: Modify provision_langchain_model() token threshold or fallback chain +4. **Add model filtering**: Extend Model.get_models_by_type() with additional filters (e.g., by provider) +5. **Implement model caching**: Wrap ModelManager methods with functools.lru_cache (be aware of kwargs mutability) + +## Usage Example + +```python +from open_notebook.ai.models import model_manager + +# Get default chat model +chat_model = await model_manager.get_default_model("chat") + +# Get specific model by ID +embedding_model = await model_manager.get_model("model:openai_embedding") + +# Get embedding model with config override +embedding_model = await model_manager.get_embedding_model(temperature=0.1) + +# Provision model for LangGraph (auto-detects large context) +from open_notebook.ai.provision import provision_langchain_model +langchain_model = await provision_langchain_model( + content=long_text, + model_id=None, # Use default + default_type="chat", + temperature=0.7 +) +``` + +--- + +## Connection Testing (connection_tester.py) + +### Purpose + +Provides functionality to test if a provider's API key is valid by making minimal API calls. Used by the API Configuration UI to validate user-entered credentials before saving. + +### test_provider_connection() + +Main entry point for testing provider connectivity. + +```python +async def test_provider_connection( + provider: str, model_type: str = "language", + config_id: Optional[str] = None +) -> Tuple[bool, str] +``` + +**Returns**: `(success: bool, message: str)` - Success status and human-readable message. + +**Flow**: +1. If `config_id` provided: Loads credential via `Credential.get(config_id)`, uses `credential.to_esperanto_config()` for provider config +2. Looks up test model from `TEST_MODELS` dict +3. For URL-based providers (ollama, openai_compatible): Tests server connectivity +4. For Azure: Tests `/openai/models` endpoint with api_version +5. For API-based providers: Creates minimal model via Esperanto and makes test call +6. Returns user-friendly error messages for common failures + +### test_individual_model() + +Tests a specific Model instance by loading its linked credential (if any) and making a minimal API call. + +### TEST_MODELS Configuration + +Maps each provider to `(model_name, model_type)` for testing: + +```python +TEST_MODELS = { + "openai": ("gpt-3.5-turbo", "language"), + "anthropic": ("claude-3-haiku-20240307", "language"), + "google": ("gemini-1.5-flash", "language"), + "groq": ("llama-3.1-8b-instant", "language"), + "voyage": ("voyage-3-lite", "embedding"), + "elevenlabs": ("eleven_multilingual_v2", "text_to_speech"), + "ollama": (None, "language"), # Dynamic + # ... more providers +} +``` + +### Special Provider Handlers + +- **`_test_ollama_connection(base_url)`**: Tests Ollama server via `/api/tags` endpoint, returns model count +- **`_test_openai_compatible_connection(base_url, api_key)`**: Tests OpenAI-compatible servers via `/models` endpoint +- **`_get_ollama_models(base_url)`**: Fetches available models from Ollama server + +### Error Message Normalization + +The tester normalizes error messages for user-friendly display: +- `401/unauthorized` -> "Invalid API key" +- `403/forbidden` -> "API key lacks required permissions" +- `rate limit` -> "Rate limited - but connection works" (success) +- `model not found` -> "API key valid (test model not available)" (success) +- Connection/timeout errors -> Helpful troubleshooting messages + +--- + +## Key Provider (key_provider.py) + +### Purpose + +Unified interface for retrieving API keys with database-first, environment-fallback strategy. Enables UI-based key management while maintaining backward compatibility with `.env` files. Used as fallback when models don't have a directly linked credential. + +### Core Functions + +#### get_api_key(provider) + +```python +async def get_api_key(provider: str) -> Optional[str] +``` + +Gets API key for a provider. Checks database (`Credential` records) first, then environment variable. + +**Fallback Chain**: +1. Query `Credential` records from database for the given provider +2. Get api_key from default credential +3. Handle `SecretStr` (call `.get_secret_value()`) vs regular strings +4. If DB value exists and is non-empty, return it +5. Otherwise, return `os.environ.get(env_var)` + +#### provision_provider_keys(provider) + +```python +async def provision_provider_keys(provider: str) -> bool +``` + +Main entry point for DB->Env fallback. Sets environment variables from database config for a provider. Called before model provisioning to ensure Esperanto can read keys from env vars. + +**Returns**: `True` if any keys were set from database. + +**Usage**: +```python +# Before creating a model, ensure DB keys are in env vars +await provision_provider_keys("openai") +model = AIFactory.create_language(model_name="gpt-4", provider="openai") +``` + +#### provision_all_keys() + +```python +async def provision_all_keys() -> dict[str, bool] +``` + +Provisions all providers at once. Useful at application startup. + +### Provider Configuration Maps + +#### PROVIDER_CONFIG (Simple Providers) + +Single-field providers with API key only: + +```python +PROVIDER_CONFIG = { + "openai": {"env_var": "OPENAI_API_KEY", "config_field": "openai_api_key"}, + "anthropic": {"env_var": "ANTHROPIC_API_KEY", "config_field": "anthropic_api_key"}, + "google": {"env_var": "GOOGLE_API_KEY", "config_field": "google_api_key"}, + "groq": {"env_var": "GROQ_API_KEY", "config_field": "groq_api_key"}, + "mistral": {"env_var": "MISTRAL_API_KEY", "config_field": "mistral_api_key"}, + "deepseek": {"env_var": "DEEPSEEK_API_KEY", "config_field": "deepseek_api_key"}, + "xai": {"env_var": "XAI_API_KEY", "config_field": "xai_api_key"}, + "openrouter": {"env_var": "OPENROUTER_API_KEY", "config_field": "openrouter_api_key"}, + "voyage": {"env_var": "VOYAGE_API_KEY", "config_field": "voyage_api_key"}, + "elevenlabs": {"env_var": "ELEVENLABS_API_KEY", "config_field": "elevenlabs_api_key"}, + "ollama": {"env_var": "OLLAMA_API_BASE", "config_field": "ollama_api_base"}, +} +``` + +#### VERTEX_CONFIG (Google Vertex AI) + +Multi-field configuration for Vertex AI: + +```python +VERTEX_CONFIG = { + "project": {"env_var": "VERTEX_PROJECT", "config_field": "vertex_project"}, + "location": {"env_var": "VERTEX_LOCATION", "config_field": "vertex_location"}, + "credentials": {"env_var": "GOOGLE_APPLICATION_CREDENTIALS", "config_field": "google_application_credentials"}, +} +``` + +#### AZURE_CONFIG (Azure OpenAI) + +Generic and mode-specific endpoints for Azure: + +```python +AZURE_CONFIG = { + "api_key": {"env_var": "AZURE_OPENAI_API_KEY", "config_field": "azure_openai_api_key"}, + "api_version": {"env_var": "AZURE_OPENAI_API_VERSION", "config_field": "azure_openai_api_version"}, + "endpoint": {"env_var": "AZURE_OPENAI_ENDPOINT", "config_field": "azure_openai_endpoint"}, + # Mode-specific endpoints + "endpoint_llm": {"env_var": "AZURE_OPENAI_ENDPOINT_LLM", "config_field": "azure_openai_endpoint_llm"}, + "endpoint_embedding": {"env_var": "AZURE_OPENAI_ENDPOINT_EMBEDDING", "config_field": "azure_openai_endpoint_embedding"}, + "endpoint_stt": {"env_var": "AZURE_OPENAI_ENDPOINT_STT", "config_field": "azure_openai_endpoint_stt"}, + "endpoint_tts": {"env_var": "AZURE_OPENAI_ENDPOINT_TTS", "config_field": "azure_openai_endpoint_tts"}, +} +``` + +#### OPENAI_COMPATIBLE_CONFIG + +Generic and mode-specific configuration for OpenAI-compatible providers: + +```python +OPENAI_COMPATIBLE_CONFIG = { + # Generic + "api_key": {"env_var": "OPENAI_COMPATIBLE_API_KEY", "config_field": "openai_compatible_api_key"}, + "base_url": {"env_var": "OPENAI_COMPATIBLE_BASE_URL", "config_field": "openai_compatible_base_url"}, + # Mode-specific: LLM, Embedding, STT, TTS + "api_key_llm": {"env_var": "OPENAI_COMPATIBLE_API_KEY_LLM", "config_field": "openai_compatible_api_key_llm"}, + "base_url_llm": {"env_var": "OPENAI_COMPATIBLE_BASE_URL_LLM", "config_field": "openai_compatible_base_url_llm"}, + # ... similar for embedding, stt, tts +} +``` + +### Internal Helper Functions + +- **`_provision_simple_provider(provider)`**: Sets single env var for simple providers +- **`_provision_vertex()`**: Sets all Vertex AI env vars +- **`_provision_azure()`**: Sets all Azure OpenAI env vars (handles SecretStr) +- **`_provision_openai_compatible()`**: Sets all OpenAI-compatible env vars + +### Integration with ModelManager + +The credential system integrates with model provisioning in two ways: + +1. **Credential-linked models** (preferred): Model has `credential` field pointing to a Credential record. `ModelManager.get_model()` calls `credential.to_esperanto_config()` and passes config directly to Esperanto's `AIFactory.create_*` methods +2. **Env var fallback**: If model has no linked credential, `provision_provider_keys(provider)` sets env vars from DB credentials; Esperanto reads from env vars +3. **ConnectionTester** loads Credential directly via `Credential.get(config_id)` for testing + +The credential-linked approach is preferred as it allows multiple credentials per provider and avoids env var mutation. diff --git a/open_notebook/ai/__init__.py b/open_notebook/ai/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..617d6373f5ca9347d660b11bd9c95a2e84b50d96 --- /dev/null +++ b/open_notebook/ai/__init__.py @@ -0,0 +1,2 @@ +# AI infrastructure module +# Contains model configuration, provisioning, and management diff --git a/open_notebook/ai/connection_tester.py b/open_notebook/ai/connection_tester.py new file mode 100644 index 0000000000000000000000000000000000000000..2825edb8cffb8b3a5d2da3924a264e2e89efb4da --- /dev/null +++ b/open_notebook/ai/connection_tester.py @@ -0,0 +1,442 @@ +""" +Connection testing for AI providers. + +This module provides functionality to test if a provider's API key is valid +by making minimal API calls to each provider, and to test individual model +configurations end-to-end. +""" +import io +import os +import struct +from typing import List, Optional, Tuple + +import httpx +from esperanto.factory import AIFactory +from loguru import logger + +from open_notebook.domain.credential import Credential + +# Test models for each provider - uses minimal/cheapest models for testing +# Format: (model_name, model_type) +TEST_MODELS = { + "openai": ("gpt-3.5-turbo", "language"), + "anthropic": ("claude-3-haiku-20240307", "language"), + "google": ("gemini-2.0-flash", "language"), + "groq": ("llama-3.1-8b-instant", "language"), + "mistral": ("mistral-small-latest", "language"), + "deepseek": ("deepseek-chat", "language"), + "xai": ("grok-beta", "language"), + "openrouter": ("openai/gpt-3.5-turbo", "language"), + "voyage": ("voyage-3-lite", "embedding"), + "elevenlabs": ("eleven_multilingual_v2", "text_to_speech"), + "ollama": (None, "language"), # Dynamic - will use first available model + # Complex providers with additional configuration + "vertex": ("gemini-2.0-flash", "language"), # Uses Google Vertex AI + "azure": ("gpt-35-turbo", "language"), # Azure OpenAI deployment name + "openai_compatible": (None, "language"), # Dynamic - will use first available model + "dashscope": ("qwen-plus", "language"), + "minimax": ("MiniMax-M2.5", "language"), +} + + +async def _test_azure_connection( + endpoint: Optional[str] = None, + api_key: Optional[str] = None, + api_version: Optional[str] = None, +) -> Tuple[bool, str]: + """ + Test Azure OpenAI connectivity by listing models. + + Azure requires deployment names which vary per user, so instead of + invoking a model, we list available models to validate credentials. + """ + test_endpoint = endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT") + test_api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY") + test_api_version = api_version or os.environ.get("AZURE_OPENAI_API_VERSION", "2024-10-21") + + if not test_endpoint: + return False, "No Azure endpoint configured" + if not test_api_key: + return False, "No Azure API key configured" + + # Strip trailing slash to avoid double-slash in URL + test_endpoint = test_endpoint.rstrip("/") + + try: + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get( + f"{test_endpoint}/openai/models?api-version={test_api_version}", + headers={"api-key": test_api_key}, + ) + + if response.status_code == 200: + data = response.json() + models = data.get("data", []) + count = len(models) + if count > 0: + names = [m.get("id", "unknown") for m in models[:3]] + name_list = ", ".join(names) + if count > 3: + name_list += f" (+{count - 3} more)" + return True, f"Connected. {count} models: {name_list}" + else: + return True, "Connected successfully (no models found)" + elif response.status_code == 401: + return False, "Invalid API key" + elif response.status_code == 403: + return False, "API key lacks required permissions" + else: + return False, f"Azure returned status {response.status_code}" + + except httpx.ConnectError: + return False, "Cannot connect to Azure endpoint. Check the URL." + except httpx.TimeoutException: + return False, "Connection timed out. Check the endpoint URL." + except Exception as e: + return False, f"Connection error: {str(e)[:100]}" + + +async def _test_ollama_connection(base_url: str) -> Tuple[bool, str]: + """Test Ollama server connectivity.""" + try: + async with httpx.AsyncClient(timeout=10.0) as client: + # Try /api/tags endpoint (standard Ollama) + response = await client.get(f"{base_url}/api/tags") + + if response.status_code == 200: + data = response.json() + models = data.get("models", []) + model_count = len(models) + + if model_count > 0: + model_names = [m.get("name", "unknown") for m in models[:3]] + model_list = ", ".join(model_names) + if model_count > 3: + model_list += f" (+{model_count - 3} more)" + return True, f"Connected. {model_count} models available: {model_list}" + else: + return True, "Connected successfully (no models listed)" + elif response.status_code == 401: + return False, "Invalid API key" + elif response.status_code == 403: + return False, "API key lacks required permissions" + else: + return False, f"Server returned status {response.status_code}" + + except httpx.ConnectError: + return False, "Cannot connect to Ollama. Check if Ollama server is running." + except httpx.TimeoutException: + return False, "Connection timed out. Check if Ollama server is accessible." + except Exception as e: + return False, f"Connection error: {str(e)[:100]}" + + +async def _test_openai_compatible_connection(base_url: str, api_key: Optional[str] = None) -> Tuple[bool, str]: + """Test OpenAI-compatible server connectivity.""" + try: + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + async with httpx.AsyncClient(timeout=10.0) as client: + # Try /models endpoint (standard OpenAI-compatible) + response = await client.get(f"{base_url}/models", headers=headers) + + if response.status_code == 200: + data = response.json() + models = data.get("data", []) + model_count = len(models) + + if model_count > 0: + model_names = [m.get("id", "unknown") for m in models[:3]] + model_list = ", ".join(model_names) + if model_count > 3: + model_list += f" (+{model_count - 3} more)" + return True, f"Connected. {model_count} models available: {model_list}" + else: + return True, "Connected successfully (no models listed)" + elif response.status_code == 401: + return False, "Invalid API key" + elif response.status_code == 403: + return False, "API key lacks required permissions" + else: + return False, f"Server returned status {response.status_code}" + + except httpx.ConnectError: + return False, "Cannot connect to server. Check the URL is correct." + except httpx.TimeoutException: + return False, "Connection timed out. Check if server is accessible." + except Exception as e: + return False, f"Connection error: {str(e)[:100]}" + +async def test_provider_connection( + provider: str, model_type: str = "language", config_id: Optional[str] = None +) -> Tuple[bool, str]: + """ + Test if a provider's API key is valid by making a minimal API call. + + Args: + provider: Provider name (openai, anthropic, etc.) + model_type: Type of model to test (language, embedding, etc.) + Note: This is overridden by TEST_MODELS if provider is in that dict. + config_id: Optional specific configuration ID to test (format: configId) + If provided, uses the configuration from ProviderConfig for this specific config. + + Returns: + Tuple of (success: bool, message: str) + """ + try: + # Get configuration - either specific config or default + api_key: Optional[str] = None + base_url: Optional[str] = None + endpoint: Optional[str] = None + api_version: Optional[str] = None + model_name: Optional[str] = None + + if config_id: + # Load specific credential from database + try: + cred = await Credential.get(config_id) + config = cred.to_esperanto_config() + api_key = config.get("api_key") + base_url = config.get("base_url") + endpoint = config.get("endpoint") + api_version = config.get("api_version") + except Exception: + return False, f"Credential not found: {config_id}" + + # Normalize provider name (handle hyphenated aliases) + normalized_provider = provider.replace("-", "_") + + # Special handling for URL-based providers (no API key, just connectivity) + if normalized_provider == "ollama": + # Use base_url from specific config, or environment variable + test_base_url = base_url or os.environ.get("OLLAMA_API_BASE", "http://localhost:11434") + return await _test_ollama_connection(test_base_url) + + if normalized_provider == "openai_compatible": + # Use base_url from specific config, or environment variable + test_base_url = base_url or os.environ.get("OPENAI_COMPATIBLE_BASE_URL") + test_api_key = api_key or os.environ.get("OPENAI_COMPATIBLE_API_KEY") + if not test_base_url: + return False, "No base URL configured for OpenAI-compatible provider" + return await _test_openai_compatible_connection(test_base_url, test_api_key) + + if normalized_provider == "azure": + # For Azure, base_url from the UI form maps to endpoint + azure_endpoint = endpoint or base_url + return await _test_azure_connection(azure_endpoint, api_key, api_version) + + # Get test model for provider + if normalized_provider not in TEST_MODELS: + return False, f"Unknown provider: {provider}" + + test_model, test_model_type = TEST_MODELS[normalized_provider] + + # Use model from config if provided, otherwise use TEST_MODELS default + model_to_use = model_name if model_name else test_model + + # For providers with dynamic model detection + if model_to_use is None: + if normalized_provider == "openai_compatible": + # OpenAI-compatible servers should already be tested via _test_openai_compatible_connection + test_base_url = base_url or os.environ.get("OPENAI_COMPATIBLE_BASE_URL", "") + test_api_key = api_key or os.environ.get("OPENAI_COMPATIBLE_API_KEY") + return await _test_openai_compatible_connection(test_base_url, test_api_key) + else: + return False, f"No test model configured for {provider}" + + # If we have a specific API key, set it in environment for this test + if api_key: + os.environ[f"{provider.upper()}_API_KEY"] = api_key + + # Try to create the model and make a minimal call + if test_model_type == "language": + model = AIFactory.create_language(model_name=model_to_use, provider=provider) + # Convert to LangChain and make a minimal call + lc_model = model.to_langchain() + await lc_model.ainvoke("Hi") + return True, "Connection successful" + + elif test_model_type == "embedding": + model = AIFactory.create_embedding(model_name=model_to_use, provider=provider) + # Embed a single short test string + await model.aembed(["test"]) + return True, "Connection successful" + + elif test_model_type == "text_to_speech": + # For TTS, we just verify the model can be created + # Making an actual TTS call would be more expensive + # Most TTS providers validate the key on model creation + AIFactory.create_text_to_speech( + model_name=model_to_use, provider=provider + ) + return True, "Connection successful (key format valid)" + + else: + return False, f"Unsupported model type for testing: {test_model_type}" + + except Exception as e: + error_msg = str(e) + + # Clean up common error messages for user-friendly display + if "401" in error_msg or "unauthorized" in error_msg.lower(): + return False, "Invalid API key" + elif "403" in error_msg or "forbidden" in error_msg.lower(): + return False, "API key lacks required permissions" + elif "rate" in error_msg.lower() and "limit" in error_msg.lower(): + # Rate limit means the key is valid but we hit limits + return True, "Rate limited - but connection works" + elif "connection" in error_msg.lower() or "network" in error_msg.lower(): + return False, "Connection error - check network/endpoint" + elif "timeout" in error_msg.lower(): + return False, "Connection timed out - check network/endpoint" + elif "not found" in error_msg.lower() and "model" in error_msg.lower(): + # Model not found but auth worked - this is actually a success for connectivity + return True, "API key valid (test model not available)" + elif provider == "ollama" and "connection refused" in error_msg.lower(): + return False, "Ollama not running - check if Ollama server is started" + else: + logger.debug(f"Test connection error for {provider}: {e}") + # Truncate long error messages + truncated = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg + return False, f"Error: {truncated}" + + +# Default voices for TTS testing per provider +# ElevenLabs excluded: uses voice_id (not name), looked up dynamically +DEFAULT_TEST_VOICES = { + "openai": "alloy", + "azure": "alloy", + "google": "Kore", + "vertex": "Kore", + "openai_compatible": "alloy", +} + + +def _generate_test_wav() -> io.BytesIO: + """Generate a minimal 0.5s silence WAV file in memory (16kHz, 16-bit mono).""" + sample_rate = 16000 + num_samples = sample_rate // 2 # 0.5 seconds + bits_per_sample = 16 + num_channels = 1 + byte_rate = sample_rate * num_channels * bits_per_sample // 8 + block_align = num_channels * bits_per_sample // 8 + data_size = num_samples * block_align + + buf = io.BytesIO() + # RIFF header + buf.write(b"RIFF") + buf.write(struct.pack(" Tuple[bool, str]: + """Normalize common error patterns into user-friendly messages.""" + lower = error_msg.lower() + + if "401" in error_msg or "unauthorized" in lower: + return False, "Invalid API key" + elif "403" in error_msg or "forbidden" in lower: + return False, "API key lacks required permissions" + elif "rate" in lower and "limit" in lower: + return True, "Rate limited - but connection works" + elif "not found" in lower and "model" in lower: + return False, "Model not found on this provider" + elif "connection" in lower or "network" in lower: + return False, "Connection error - check network/endpoint" + elif "timeout" in lower: + return False, "Connection timed out - check network/endpoint" + + return False, error_msg + + +async def test_individual_model(model) -> Tuple[bool, str]: + """ + Test a specific model configuration end-to-end by making a real API call. + + Args: + model: A Model instance (from open_notebook.ai.models) + + Returns: + Tuple of (success: bool, message: str) + """ + from open_notebook.ai.models import ModelManager + + try: + manager = ModelManager() + esp_model = await manager.get_model(model.id) + + if esp_model is None: + return False, "Could not create model instance" + + if model.type == "language": + response = await esp_model.achat_complete( + messages=[{"role": "user", "content": "Hi!"}] + ) + text = response.content[:100] if response.content else "(empty response)" + return True, f"Response: {text}" + + elif model.type == "embedding": + result = await esp_model.aembed(["This is a test."]) + if result and len(result) > 0: + dims = len(result[0]) + return True, f"Embedding dimensions: {dims}" + return True, "Embedding successful" + + elif model.type == "text_to_speech": + # For ElevenLabs, look up first available voice (API uses voice_id, not name) + voice = DEFAULT_TEST_VOICES.get(model.provider) + if not voice and hasattr(esp_model, "available_voices"): + try: + voices = esp_model.available_voices + if voices: + voice = next(iter(voices.keys())) + except Exception: + pass + if not voice: + voice = "alloy" # fallback + + result = await esp_model.agenerate_speech( + text="Hello from Open Notebook", voice=voice + ) + if result and hasattr(result, "content"): + size = len(result.content) + return True, f"Audio generated: {size} bytes" + return True, "Speech generation successful" + + elif model.type == "speech_to_text": + audio_file = _generate_test_wav() + result = await esp_model.atranscribe( + audio_file=audio_file, language="en" + ) + text = str(result.text) if hasattr(result, "text") else str(result) + return True, f"Transcription: {text[:100]}" + + else: + return False, f"Unsupported model type: {model.type}" + + except Exception as e: + error_msg = str(e) + success, normalized = _normalize_error_message(error_msg) + if success: + return True, normalized + logger.debug(f"Test individual model error for {model.id}: {e}") + return False, normalized diff --git a/open_notebook/ai/key_provider.py b/open_notebook/ai/key_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..d7a621deb3f0a159b609c969370b259cc9c9e230 --- /dev/null +++ b/open_notebook/ai/key_provider.py @@ -0,0 +1,305 @@ +""" +API Key Provider - Database-first with environment fallback. + +This module provides a unified interface for retrieving API keys and provider +configuration. It reads from Credential records (individual per-provider +credentials) and falls back to environment variables for backward compatibility. + +Usage: + from open_notebook.ai.key_provider import provision_provider_keys + + # Call before model provisioning to set env vars from DB + await provision_provider_keys("openai") +""" + +import os +from typing import Optional + +from loguru import logger + +from open_notebook.domain.credential import Credential + + +# ============================================================================= +# Provider Configuration Mapping +# ============================================================================= +# Maps provider names to their environment variable names. +# This is the single source of truth for provider-to-env-var mapping. + +PROVIDER_CONFIG = { + # Simple providers (just API key) + "openai": { + "env_var": "OPENAI_API_KEY", + }, + "anthropic": { + "env_var": "ANTHROPIC_API_KEY", + }, + "google": { + "env_var": "GOOGLE_API_KEY", + }, + "groq": { + "env_var": "GROQ_API_KEY", + }, + "mistral": { + "env_var": "MISTRAL_API_KEY", + }, + "deepseek": { + "env_var": "DEEPSEEK_API_KEY", + }, + "xai": { + "env_var": "XAI_API_KEY", + }, + "openrouter": { + "env_var": "OPENROUTER_API_KEY", + }, + "voyage": { + "env_var": "VOYAGE_API_KEY", + }, + "elevenlabs": { + "env_var": "ELEVENLABS_API_KEY", + }, + # URL-based providers + "ollama": { + "env_var": "OLLAMA_API_BASE", + }, + "dashscope": { + "env_var": "DASHSCOPE_API_KEY", + }, + "minimax": { + "env_var": "MINIMAX_API_KEY", + }, +} + + +async def _get_default_credential(provider: str) -> Optional[Credential]: + """Get the first credential for a provider from the database.""" + try: + credentials = await Credential.get_by_provider(provider) + if credentials: + return credentials[0] + except Exception as e: + logger.debug(f"Could not load credential from database for {provider}: {e}") + return None + + +async def get_api_key(provider: str) -> Optional[str]: + """ + Get API key for a provider. Checks database first, then env var. + + Args: + provider: Provider name (openai, anthropic, etc.) + + Returns: + API key string or None if not configured + """ + cred = await _get_default_credential(provider) + if cred and cred.api_key: + logger.debug(f"Using {provider} API key from Credential") + return cred.api_key.get_secret_value() + + # Fall back to environment variable + config_info = PROVIDER_CONFIG.get(provider.lower()) + if config_info: + env_value = os.environ.get(config_info["env_var"]) + if env_value: + logger.debug(f"Using {provider} API key from environment variable") + return env_value + + return None + + +async def _provision_simple_provider(provider: str) -> bool: + """ + Set environment variable for a simple provider from DB config. + + Returns: + True if key was set from database, False otherwise + """ + provider_lower = provider.lower() + config_info = PROVIDER_CONFIG.get(provider_lower) + if not config_info: + return False + + env_var = config_info["env_var"] + + cred = await _get_default_credential(provider_lower) + if not cred: + return False + + # Set API key / primary env var + if cred.api_key: + os.environ[env_var] = cred.api_key.get_secret_value() + logger.debug(f"Set {env_var} from Credential") + + # Set base URL if present + if cred.base_url: + provider_upper = provider_lower.upper() + os.environ[f"{provider_upper}_API_BASE"] = cred.base_url + logger.debug(f"Set {provider_upper}_API_BASE from Credential") + + return True + + +async def _provision_vertex() -> bool: + """ + Set environment variables for Google Vertex AI from DB config. + + Returns: + True if any keys were set from database + """ + any_set = False + + cred = await _get_default_credential("vertex") + if not cred: + return False + + if cred.project: + os.environ["VERTEX_PROJECT"] = cred.project + logger.debug("Set VERTEX_PROJECT from Credential") + any_set = True + if cred.location: + os.environ["VERTEX_LOCATION"] = cred.location + logger.debug("Set VERTEX_LOCATION from Credential") + any_set = True + if cred.credentials_path: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cred.credentials_path + logger.debug("Set GOOGLE_APPLICATION_CREDENTIALS from Credential") + any_set = True + + return any_set + + +async def _provision_azure() -> bool: + """ + Set environment variables for Azure OpenAI from DB config. + + Returns: + True if any keys were set from database + """ + any_set = False + + cred = await _get_default_credential("azure") + if not cred: + return False + + if cred.api_key: + os.environ["AZURE_OPENAI_API_KEY"] = cred.api_key.get_secret_value() + logger.debug("Set AZURE_OPENAI_API_KEY from Credential") + any_set = True + if cred.api_version: + os.environ["AZURE_OPENAI_API_VERSION"] = cred.api_version + logger.debug("Set AZURE_OPENAI_API_VERSION from Credential") + any_set = True + # For Azure, base_url from the UI form maps to endpoint + azure_endpoint = cred.endpoint or cred.base_url + if azure_endpoint: + os.environ["AZURE_OPENAI_ENDPOINT"] = azure_endpoint + logger.debug("Set AZURE_OPENAI_ENDPOINT from Credential") + any_set = True + if cred.endpoint_llm: + os.environ["AZURE_OPENAI_ENDPOINT_LLM"] = cred.endpoint_llm + logger.debug("Set AZURE_OPENAI_ENDPOINT_LLM from Credential") + any_set = True + if cred.endpoint_embedding: + os.environ["AZURE_OPENAI_ENDPOINT_EMBEDDING"] = cred.endpoint_embedding + logger.debug("Set AZURE_OPENAI_ENDPOINT_EMBEDDING from Credential") + any_set = True + if cred.endpoint_stt: + os.environ["AZURE_OPENAI_ENDPOINT_STT"] = cred.endpoint_stt + logger.debug("Set AZURE_OPENAI_ENDPOINT_STT from Credential") + any_set = True + if cred.endpoint_tts: + os.environ["AZURE_OPENAI_ENDPOINT_TTS"] = cred.endpoint_tts + logger.debug("Set AZURE_OPENAI_ENDPOINT_TTS from Credential") + any_set = True + + return any_set + + +async def _provision_openai_compatible() -> bool: + """ + Set environment variables for OpenAI-Compatible providers from DB config. + + Returns: + True if any keys were set from database + """ + any_set = False + + cred = await _get_default_credential("openai_compatible") + if not cred: + return False + + if cred.api_key: + os.environ["OPENAI_COMPATIBLE_API_KEY"] = cred.api_key.get_secret_value() + logger.debug("Set OPENAI_COMPATIBLE_API_KEY from Credential") + any_set = True + if cred.base_url: + os.environ["OPENAI_COMPATIBLE_BASE_URL"] = cred.base_url + logger.debug("Set OPENAI_COMPATIBLE_BASE_URL from Credential") + any_set = True + + return any_set + + +async def provision_provider_keys(provider: str) -> bool: + """ + Provision environment variables from database for a specific provider. + + This function checks if the provider has a Credential record stored in the + database and sets the corresponding environment variables. If the database + doesn't have the configuration, existing environment variables remain unchanged. + + This is the main entry point for the DB->Env fallback mechanism. + + Args: + provider: Provider name (openai, anthropic, azure, vertex, + openai-compatible, etc.) + + Returns: + True if any keys were set from database, False otherwise + + Example: + # Before provisioning a model, ensure DB keys are in env vars + await provision_provider_keys("openai") + model = AIFactory.create_language(model_name="gpt-4", provider="openai") + """ + # Normalize provider name + provider_lower = provider.lower() + + # Handle complex providers with multiple config fields + if provider_lower == "vertex": + return await _provision_vertex() + elif provider_lower == "azure": + return await _provision_azure() + elif provider_lower in ("openai-compatible", "openai_compatible"): + return await _provision_openai_compatible() + + # Handle simple providers + return await _provision_simple_provider(provider_lower) + + +async def provision_all_keys() -> dict[str, bool]: + """ + Provision environment variables from database for all providers. + + NOTE: This function is deprecated for request-time use because it can leave + stale env vars after key deletion. Keys should only be provisioned at startup + or via provision_provider_keys() for specific providers. + + Useful at application startup to load all DB-stored keys into environment. + + Returns: + Dict mapping provider names to whether keys were set from DB + """ + results: dict[str, bool] = {} + + # Simple providers + for provider in PROVIDER_CONFIG.keys(): + results[provider] = await provision_provider_keys(provider) + + # Complex providers + results["vertex"] = await provision_provider_keys("vertex") + results["azure"] = await provision_provider_keys("azure") + results["openai_compatible"] = await provision_provider_keys("openai_compatible") + + return results diff --git a/open_notebook/ai/model_discovery.py b/open_notebook/ai/model_discovery.py new file mode 100644 index 0000000000000000000000000000000000000000..7270843311bb49c920b0537ad501f5ff86b081e4 --- /dev/null +++ b/open_notebook/ai/model_discovery.py @@ -0,0 +1,836 @@ +""" +Model Discovery - Automatic model fetching from AI providers. + +This module provides functionality to discover available models from configured +AI providers and automatically register them in the database. +""" + +import asyncio +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import httpx +from loguru import logger + +from open_notebook.ai.models import Model +from open_notebook.domain.credential import Credential +from open_notebook.database.repository import repo_query + + +@dataclass +class DiscoveredModel: + """Represents a model discovered from a provider.""" + + name: str + provider: str + model_type: str # language, embedding, speech_to_text, text_to_speech + description: Optional[str] = None + + +# ============================================================================= +# Provider-Specific Model Type Classification +# ============================================================================= +# These mappings help classify models by their capabilities based on naming patterns + +OPENAI_MODEL_TYPES = { + "language": [ + "gpt-4", + "gpt-3.5", + "o1", + "o3", + "chatgpt", + "text-davinci", + "davinci", + "curie", + "babbage", + "ada", + ], + "embedding": ["text-embedding", "embedding"], + "speech_to_text": ["whisper"], + "text_to_speech": ["tts"], +} + +ANTHROPIC_MODELS = { + # Static list since Anthropic doesn't have a model listing API + "language": [ + "claude-opus-4-20250514", + "claude-sonnet-4-20250514", + "claude-3-5-sonnet-20241022", + "claude-3-5-haiku-20241022", + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", + ], +} + +GOOGLE_MODEL_TYPES = { + "language": ["gemini", "palm", "bison", "chat"], + "embedding": ["embedding", "textembedding"], +} + +OLLAMA_MODEL_TYPES = { + # Ollama models can do multiple things, classify by common names + "language": [ + "llama", + "mistral", + "mixtral", + "codellama", + "phi", + "gemma", + "qwen", + "deepseek", + "vicuna", + "falcon", + "orca", + "neural", + "dolphin", + "openchat", + "starling", + "solar", + "yi", + "nous", + "wizard", + "zephyr", + "tinyllama", + ], + "embedding": ["nomic-embed", "mxbai-embed", "all-minilm", "bge-", "e5-"], +} + +MISTRAL_MODEL_TYPES = { + "language": [ + "mistral", + "mixtral", + "codestral", + "ministral", + "pixtral", + "open-mistral", + "open-mixtral", + ], + "embedding": ["mistral-embed"], +} + +GROQ_MODEL_TYPES = { + "language": ["llama", "mixtral", "gemma", "whisper"], + "speech_to_text": ["whisper"], +} + +DEEPSEEK_MODEL_TYPES = { + "language": ["deepseek-chat", "deepseek-reasoner", "deepseek-coder"], +} + +XAI_MODEL_TYPES = { + "language": ["grok"], +} + +VOYAGE_MODEL_TYPES = { + "embedding": ["voyage"], +} + +ELEVENLABS_MODEL_TYPES = { + "text_to_speech": ["eleven"], +} + +DASHSCOPE_MODEL_TYPES = { + "language": ["qwen"], +} + +MINIMAX_MODEL_TYPES = { + "language": ["minimax", "abab"], +} + + +def classify_model_type(model_name: str, provider: str) -> str: + """ + Classify a model into a type based on its name and provider. + + Returns one of: language, embedding, speech_to_text, text_to_speech + """ + name_lower = model_name.lower() + + type_mappings = { + "openai": OPENAI_MODEL_TYPES, + "google": GOOGLE_MODEL_TYPES, + "ollama": OLLAMA_MODEL_TYPES, + "mistral": MISTRAL_MODEL_TYPES, + "groq": GROQ_MODEL_TYPES, + "deepseek": DEEPSEEK_MODEL_TYPES, + "xai": XAI_MODEL_TYPES, + "voyage": VOYAGE_MODEL_TYPES, + "elevenlabs": ELEVENLABS_MODEL_TYPES, + "dashscope": DASHSCOPE_MODEL_TYPES, + "minimax": MINIMAX_MODEL_TYPES, + } + + mapping = type_mappings.get(provider, {}) + + # Check each type in order of specificity + for model_type in ["speech_to_text", "text_to_speech", "embedding", "language"]: + patterns = mapping.get(model_type, []) + for pattern in patterns: + if pattern in name_lower: + return model_type + + # Default to language for unknown models + return "language" + + +# ============================================================================= +# Provider-Specific Model Discovery Functions +# ============================================================================= + + +async def discover_openai_models() -> List[DiscoveredModel]: + """Fetch available models from OpenAI API.""" + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.openai.com/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "openai") + models.append( + DiscoveredModel( + name=model_id, + provider="openai", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover OpenAI models: {e}") + + return models + + +async def discover_anthropic_models() -> List[DiscoveredModel]: + """Return static list of Anthropic models (no discovery API available).""" + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + return [] + + # Anthropic doesn't have a model listing API, so we use a static list + models = [] + for model_name in ANTHROPIC_MODELS.get("language", []): + models.append( + DiscoveredModel( + name=model_name, + provider="anthropic", + model_type="language", + ) + ) + return models + + +async def discover_google_models() -> List[DiscoveredModel]: + """Fetch available models from Google Gemini API.""" + api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + # Build URL without logging the key to avoid exposure + url = "https://generativelanguage.googleapis.com/v1/models" + headers = {"X-Goog-Api-Key": api_key} + response = await client.get(url, headers=headers, timeout=30.0) + response.raise_for_status() + data = response.json() + + for model in data.get("models", []): + # Google returns full path like "models/gemini-1.5-flash" + model_name = model.get("name", "").replace("models/", "") + if model_name: + model_type = classify_model_type(model_name, "google") + # Check supported generation methods for better classification + methods = model.get("supportedGenerationMethods", []) + if "embedContent" in methods: + model_type = "embedding" + elif "generateContent" in methods: + model_type = "language" + + models.append( + DiscoveredModel( + name=model_name, + provider="google", + model_type=model_type, + description=model.get("displayName"), + ) + ) + except Exception as e: + # Log without exposing the API key in the message + logger.warning(f"Failed to discover Google models: {type(e).__name__}") + + return models + + +async def discover_ollama_models() -> List[DiscoveredModel]: + """Fetch available models from local Ollama instance.""" + base_url = os.environ.get("OLLAMA_API_BASE", "http://localhost:11434") + if not base_url: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{base_url}/api/tags", + timeout=10.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("models", []): + model_name = model.get("name", "") + if model_name: + model_type = classify_model_type(model_name, "ollama") + models.append( + DiscoveredModel( + name=model_name, + provider="ollama", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover Ollama models: {e}") + + return models + + +async def discover_groq_models() -> List[DiscoveredModel]: + """Fetch available models from Groq API.""" + api_key = os.environ.get("GROQ_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.groq.com/openai/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "groq") + models.append( + DiscoveredModel( + name=model_id, + provider="groq", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover Groq models: {e}") + + return models + + +async def discover_mistral_models() -> List[DiscoveredModel]: + """Fetch available models from Mistral API.""" + api_key = os.environ.get("MISTRAL_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.mistral.ai/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "mistral") + # Check capabilities if available + capabilities = model.get("capabilities", {}) + if capabilities.get("completion_chat"): + model_type = "language" + + models.append( + DiscoveredModel( + name=model_id, + provider="mistral", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover Mistral models: {e}") + + return models + + +async def discover_deepseek_models() -> List[DiscoveredModel]: + """Fetch available models from DeepSeek API.""" + api_key = os.environ.get("DEEPSEEK_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.deepseek.com/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "deepseek") + models.append( + DiscoveredModel( + name=model_id, + provider="deepseek", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover DeepSeek models: {e}") + + return models + + +async def discover_xai_models() -> List[DiscoveredModel]: + """Fetch available models from xAI API.""" + api_key = os.environ.get("XAI_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.x.ai/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "xai") + models.append( + DiscoveredModel( + name=model_id, + provider="xai", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover xAI models: {e}") + + return models + + +async def discover_openrouter_models() -> List[DiscoveredModel]: + """Fetch available models from OpenRouter API.""" + api_key = os.environ.get("OPENROUTER_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://openrouter.ai/api/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + # OpenRouter models are typically language models + models.append( + DiscoveredModel( + name=model_id, + provider="openrouter", + model_type="language", + description=model.get("name"), + ) + ) + except Exception as e: + logger.warning(f"Failed to discover OpenRouter models: {e}") + + return models + + +async def discover_voyage_models() -> List[DiscoveredModel]: + """Return static list of Voyage AI models (embedding only).""" + api_key = os.environ.get("VOYAGE_API_KEY") + if not api_key: + return [] + + # Voyage AI specializes in embeddings + voyage_models = [ + "voyage-3", + "voyage-3-lite", + "voyage-code-3", + "voyage-finance-2", + "voyage-law-2", + "voyage-multilingual-2", + ] + + return [ + DiscoveredModel(name=m, provider="voyage", model_type="embedding") + for m in voyage_models + ] + + +async def discover_elevenlabs_models() -> List[DiscoveredModel]: + """Return static list of ElevenLabs TTS models.""" + api_key = os.environ.get("ELEVENLABS_API_KEY") + if not api_key: + return [] + + # ElevenLabs specializes in TTS + elevenlabs_models = [ + "eleven_multilingual_v2", + "eleven_turbo_v2_5", + "eleven_turbo_v2", + "eleven_monolingual_v1", + "eleven_multilingual_v1", + ] + + return [ + DiscoveredModel(name=m, provider="elevenlabs", model_type="text_to_speech") + for m in elevenlabs_models + ] + + +async def discover_dashscope_models() -> List[DiscoveredModel]: + """Fetch available models from DashScope (Qwen) API.""" + api_key = os.environ.get("DASHSCOPE_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://dashscope.aliyuncs.com/compatible-mode/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "dashscope") + models.append( + DiscoveredModel( + name=model_id, + provider="dashscope", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover DashScope models: {e}") + + return models + + +async def discover_minimax_models() -> List[DiscoveredModel]: + """Fetch available models from MiniMax API.""" + api_key = os.environ.get("MINIMAX_API_KEY") + if not api_key: + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.minimax.io/v1/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + model_type = classify_model_type(model_id, "minimax") + models.append( + DiscoveredModel( + name=model_id, + provider="minimax", + model_type=model_type, + ) + ) + except Exception as e: + logger.warning(f"Failed to discover MiniMax models: {e}") + + return models + + +async def discover_openai_compatible_models() -> List[DiscoveredModel]: + """ + Fetch available models from an OpenAI-compatible API endpoint. + Uses the configured base_url from the database or environment variable. + """ + api_key = None + base_url = None + + # Try to get config from Credential database first + try: + credentials = await Credential.get_by_provider("openai_compatible") + if credentials: + cred = credentials[0] + config = cred.to_esperanto_config() + api_key = config.get("api_key") + base_url = config.get("base_url", "").rstrip("/") + except Exception as e: + logger.warning(f"Failed to read openai_compatible config from Credential: {e}") + + # Fall back to environment variables + if not api_key: + api_key = os.environ.get("OPENAI_COMPATIBLE_API_KEY") + if not base_url: + base_url = os.environ.get("OPENAI_COMPATIBLE_BASE_URL", "").rstrip("/") + + if not base_url: + logger.warning("No base_url configured for openai_compatible provider") + return [] + + models = [] + try: + async with httpx.AsyncClient() as client: + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + response = await client.get( + f"{base_url}/models", + headers=headers, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + + for model in data.get("data", []): + model_id = model.get("id", "") + if model_id: + # Classify based on model name patterns + model_type = classify_model_type(model_id, "openai") + models.append( + DiscoveredModel( + name=model_id, + provider="openai_compatible", + model_type=model_type, + ) + ) + except httpx.HTTPStatusError as e: + logger.warning(f"Failed to discover openai_compatible models: HTTP {e.response.status_code}") + except Exception as e: + logger.warning(f"Failed to discover openai_compatible models: {e}") + + return models + + +# ============================================================================= +# Main Discovery Functions +# ============================================================================= + +# Map provider names to their discovery functions +PROVIDER_DISCOVERY_FUNCTIONS = { + "openai": discover_openai_models, + "anthropic": discover_anthropic_models, + "google": discover_google_models, + "ollama": discover_ollama_models, + "groq": discover_groq_models, + "mistral": discover_mistral_models, + "deepseek": discover_deepseek_models, + "xai": discover_xai_models, + "openrouter": discover_openrouter_models, + "voyage": discover_voyage_models, + "elevenlabs": discover_elevenlabs_models, + "openai_compatible": discover_openai_compatible_models, + "dashscope": discover_dashscope_models, + "minimax": discover_minimax_models, + "azure": None, # Azure requires credential-based discovery (different auth) + "vertex": None, # Vertex requires credential-based discovery (service account) +} + + +async def discover_provider_models(provider: str) -> List[DiscoveredModel]: + """ + Discover available models for a specific provider. + + Args: + provider: Provider name (openai, anthropic, etc.) + + Returns: + List of discovered models + """ + discover_func = PROVIDER_DISCOVERY_FUNCTIONS.get(provider) + if discover_func is None: + if provider in PROVIDER_DISCOVERY_FUNCTIONS: + logger.info( + f"Provider '{provider}' requires credential-based discovery. " + f"Use the /credentials/{{id}}/discover endpoint instead." + ) + else: + logger.warning(f"No discovery function for provider: {provider}") + return [] + + return await discover_func() + + +async def sync_provider_models( + provider: str, auto_register: bool = True +) -> Tuple[int, int, int]: + """ + Sync models for a provider: discover and optionally register in database. + + Args: + provider: Provider name + auto_register: If True, automatically create Model records in database + + Returns: + Tuple of (discovered_count, new_count, existing_count) + """ + discovered = await discover_provider_models(provider) + discovered_count = len(discovered) + new_count = 0 + existing_count = 0 + + if not auto_register: + return discovered_count, 0, 0 + + if not discovered: + return 0, 0, 0 + + # Batch fetch existing models to avoid N+1 query pattern + try: + existing_models = await repo_query( + "SELECT string::lowercase(name) as name, string::lowercase(type) as type FROM model " + "WHERE string::lowercase(provider) = $provider", + {"provider": provider.lower()}, + ) + # Create a set of (name, type) tuples for O(1) lookup + existing_keys = set() + for m in existing_models: + existing_keys.add((m.get("name", ""), m.get("type", ""))) + except Exception as e: + logger.warning(f"Failed to fetch existing models for {provider}: {e}") + existing_keys = set() + + for model in discovered: + model_key = (model.name.lower(), model.model_type.lower()) + + # Check if model already exists using pre-fetched data + if model_key in existing_keys: + existing_count += 1 + continue + + # Create new model + try: + new_model = Model( + name=model.name, + provider=model.provider, + type=model.model_type, + ) + await new_model.save() + new_count += 1 + logger.info(f"Registered new model: {model.provider}/{model.name} ({model.model_type})") + except Exception as e: + logger.warning(f"Failed to register model {model.name}: {e}") + + logger.info( + f"Synced {provider}: {discovered_count} discovered, " + f"{new_count} new, {existing_count} existing" + ) + return discovered_count, new_count, existing_count + + +async def sync_all_providers() -> Dict[str, Tuple[int, int, int]]: + """ + Sync models for all configured providers. + + Returns: + Dict mapping provider names to (discovered, new, existing) tuples + """ + results = {} + + # Run discovery for all providers in parallel + tasks = [] + providers = list(PROVIDER_DISCOVERY_FUNCTIONS.keys()) + + for provider in providers: + tasks.append(sync_provider_models(provider, auto_register=True)) + + task_results = await asyncio.gather(*tasks, return_exceptions=True) + + for provider, result in zip(providers, task_results): + if isinstance(result, Exception): + logger.error(f"Error syncing {provider}: {result}") + results[provider] = (0, 0, 0) + else: + results[provider] = result + + return results + + +async def get_provider_model_count(provider: str) -> Dict[str, int]: + """ + Get count of registered models for a provider, grouped by type. + + Args: + provider: Provider name (case-insensitive) + + Returns: + Dict mapping model type to count + """ + # Use case-insensitive comparison by lowercasing the provider + result = await repo_query( + "SELECT type, count() as count FROM model WHERE string::lowercase(provider) = string::lowercase($provider) GROUP BY type", + {"provider": provider}, + ) + + counts = { + "language": 0, + "embedding": 0, + "speech_to_text": 0, + "text_to_speech": 0, + } + + for row in result: + model_type = row.get("type") + count = row.get("count", 0) + if model_type in counts: + counts[model_type] = count + + return counts diff --git a/open_notebook/ai/models.py b/open_notebook/ai/models.py new file mode 100644 index 0000000000000000000000000000000000000000..076340b0496898e8f0119dbbfc6edbabb393fa19 --- /dev/null +++ b/open_notebook/ai/models.py @@ -0,0 +1,267 @@ +from typing import Any, ClassVar, Dict, Optional, Union + +from esperanto import ( + AIFactory, + EmbeddingModel, + LanguageModel, + SpeechToTextModel, + TextToSpeechModel, +) +from loguru import logger + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.base import ObjectModel, RecordModel +from open_notebook.exceptions import ConfigurationError + +ModelType = Union[LanguageModel, EmbeddingModel, SpeechToTextModel, TextToSpeechModel] + + +class Model(ObjectModel): + table_name: ClassVar[str] = "model" + nullable_fields: ClassVar[set[str]] = {"credential"} + name: str + provider: str + type: str + credential: Optional[str] = None + + @classmethod + async def get_models_by_type(cls, model_type): + models = await repo_query( + "SELECT * FROM model WHERE type=$model_type;", {"model_type": model_type} + ) + return [Model(**model) for model in models] + + @classmethod + async def get_by_credential(cls, credential_id: str): + """Get all models linked to a specific credential.""" + models = await repo_query( + "SELECT * FROM model WHERE credential=$cred_id;", + {"cred_id": ensure_record_id(credential_id)}, + ) + return [Model(**model) for model in models] + + def _prepare_save_data(self) -> Dict[str, Any]: + data = super()._prepare_save_data() + if data.get("credential"): + data["credential"] = ensure_record_id(data["credential"]) + return data + + async def get_credential_obj(self): + """Get the Credential object linked to this model, if any.""" + if not self.credential: + return None + from open_notebook.domain.credential import Credential + + try: + return await Credential.get(self.credential) + except Exception: + logger.warning(f"Could not load credential {self.credential} for model {self.id}") + return None + + +class DefaultModels(RecordModel): + record_id: ClassVar[str] = "open_notebook:default_models" + default_chat_model: Optional[str] = None + default_transformation_model: Optional[str] = None + large_context_model: Optional[str] = None + default_text_to_speech_model: Optional[str] = None + default_speech_to_text_model: Optional[str] = None + # default_vision_model: Optional[str] + default_embedding_model: Optional[str] = None + default_tools_model: Optional[str] = None + + @classmethod + async def get_instance(cls) -> "DefaultModels": + """Always fetch fresh defaults from database (override parent caching behavior)""" + result = await repo_query( + "SELECT * FROM ONLY $record_id", + {"record_id": ensure_record_id(cls.record_id)}, + ) + + if result: + if isinstance(result, list) and len(result) > 0: + data = result[0] + elif isinstance(result, dict): + data = result + else: + data = {} + else: + data = {} + + # Create new instance with fresh data (bypass singleton cache) + instance = object.__new__(cls) + object.__setattr__(instance, "__dict__", {}) + super(RecordModel, instance).__init__(**data) + return instance + + +class ModelManager: + def __init__(self): + pass # No caching needed + + async def get_model(self, model_id: str, **kwargs) -> Optional[ModelType]: + """Get a model by ID. Esperanto will cache the actual model instance.""" + if not model_id: + return None + + try: + model: Model = await Model.get(model_id) + except Exception: + raise ConfigurationError(f"Model with ID {model_id} not found") + + if not model.type or model.type not in [ + "language", + "embedding", + "speech_to_text", + "text_to_speech", + ]: + raise ConfigurationError(f"Invalid model type: {model.type}") + + # Build config from credential if linked, otherwise fall back to env vars + config: dict = {} + if model.credential: + credential = await model.get_credential_obj() + if credential: + config = credential.to_esperanto_config() + logger.debug( + f"Using credential '{credential.name}' for model {model.name}" + ) + else: + logger.warning( + f"Model {model.id} has credential {model.credential} but it could not be loaded. " + f"Falling back to env vars." + ) + # Fall back to env var provisioning + from open_notebook.ai.key_provider import provision_provider_keys + + await provision_provider_keys(model.provider) + else: + # No credential linked - use env var fallback + from open_notebook.ai.key_provider import provision_provider_keys + + await provision_provider_keys(model.provider) + + # Merge any additional kwargs (e.g. temperature) + config.update(kwargs) + + # Normalize provider name: DB stores underscores but Esperanto expects hyphens + provider = model.provider.replace("_", "-") + + # Create model based on type (Esperanto will cache the instance) + if model.type == "language": + return AIFactory.create_language( + model_name=model.name, + provider=provider, + config=config, + ) + elif model.type == "embedding": + return AIFactory.create_embedding( + model_name=model.name, + provider=provider, + config=config, + ) + elif model.type == "speech_to_text": + return AIFactory.create_speech_to_text( + model_name=model.name, + provider=provider, + config=config, + ) + elif model.type == "text_to_speech": + return AIFactory.create_text_to_speech( + model_name=model.name, + provider=provider, + config=config, + ) + else: + raise ConfigurationError(f"Invalid model type: {model.type}") + + async def get_defaults(self) -> DefaultModels: + """Get the default models configuration from database""" + defaults = await DefaultModels.get_instance() + if not defaults: + raise RuntimeError("Failed to load default models configuration") + return defaults + + async def get_speech_to_text(self, **kwargs) -> Optional[SpeechToTextModel]: + """Get the default speech-to-text model""" + defaults = await self.get_defaults() + model_id = defaults.default_speech_to_text_model + if not model_id: + return None + model = await self.get_model(model_id, **kwargs) + assert model is None or isinstance(model, SpeechToTextModel), ( + f"Expected SpeechToTextModel but got {type(model)}" + ) + return model + + async def get_text_to_speech(self, **kwargs) -> Optional[TextToSpeechModel]: + """Get the default text-to-speech model""" + defaults = await self.get_defaults() + model_id = defaults.default_text_to_speech_model + if not model_id: + return None + model = await self.get_model(model_id, **kwargs) + assert model is None or isinstance(model, TextToSpeechModel), ( + f"Expected TextToSpeechModel but got {type(model)}" + ) + return model + + async def get_embedding_model(self, **kwargs) -> Optional[EmbeddingModel]: + """Get the default embedding model""" + defaults = await self.get_defaults() + model_id = defaults.default_embedding_model + if not model_id: + return None + model = await self.get_model(model_id, **kwargs) + assert model is None or isinstance(model, EmbeddingModel), ( + f"Expected EmbeddingModel but got {type(model)}" + ) + return model + + async def get_default_model(self, model_type: str, **kwargs) -> Optional[ModelType]: + """ + Get the default model for a specific type. + + Args: + model_type: The type of model to retrieve (e.g., 'chat', 'embedding', etc.) + **kwargs: Additional arguments to pass to the model constructor + """ + defaults = await self.get_defaults() + model_id = None + + if model_type == "chat": + model_id = defaults.default_chat_model + elif model_type == "transformation": + model_id = ( + defaults.default_transformation_model or defaults.default_chat_model + ) + elif model_type == "tools": + model_id = defaults.default_tools_model or defaults.default_chat_model + elif model_type == "embedding": + model_id = defaults.default_embedding_model + elif model_type == "text_to_speech": + model_id = defaults.default_text_to_speech_model + elif model_type == "speech_to_text": + model_id = defaults.default_speech_to_text_model + elif model_type == "large_context": + model_id = defaults.large_context_model + + if not model_id: + logger.warning( + f"No default model configured for type '{model_type}'. " + f"Please go to Settings → Models and set a default model." + ) + return None + + try: + return await self.get_model(model_id, **kwargs) + except (ValueError, ConfigurationError) as e: + logger.error( + f"Failed to load default model for type '{model_type}': {e}. " + f"The configured model_id '{model_id}' may have been deleted or misconfigured. " + f"Please go to Settings → Models and reconfigure the default model." + ) + return None + + +model_manager = ModelManager() diff --git a/open_notebook/ai/provision.py b/open_notebook/ai/provision.py new file mode 100644 index 0000000000000000000000000000000000000000..f3dc4f614cbf64e2ad5f98933890360909be63a4 --- /dev/null +++ b/open_notebook/ai/provision.py @@ -0,0 +1,61 @@ +from esperanto import LanguageModel +from langchain_core.language_models.chat_models import BaseChatModel +from loguru import logger + +from open_notebook.ai.models import model_manager +from open_notebook.exceptions import ConfigurationError +from open_notebook.utils import token_count + + +async def provision_langchain_model( + content, model_id, default_type, **kwargs +) -> BaseChatModel: + """ + Returns the best model to use based on the context size and on whether there is a specific model being requested in Config. + If context > 105_000, returns the large_context_model + If model_id is specified in Config, returns that model + Otherwise, returns the default model for the given type + """ + tokens = token_count(content) + model = None + selection_reason = "" + + if tokens > 105_000: + selection_reason = f"large_context (content has {tokens} tokens)" + logger.debug( + f"Using large context model because the content has {tokens} tokens" + ) + model = await model_manager.get_default_model("large_context", **kwargs) + elif model_id: + selection_reason = f"explicit model_id={model_id}" + model = await model_manager.get_model(model_id, **kwargs) + else: + selection_reason = f"default for type={default_type}" + model = await model_manager.get_default_model(default_type, **kwargs) + + logger.debug(f"Using model: {model}") + + if model is None: + logger.error( + f"Model provisioning failed: No model found. " + f"Selection reason: {selection_reason}. " + f"model_id={model_id}, default_type={default_type}. " + f"Please check Settings → Models and ensure a default model is configured for '{default_type}'." + ) + raise ConfigurationError( + f"No model configured for {selection_reason}. " + f"Please go to Settings → Models and configure a default model for '{default_type}'." + ) + + if not isinstance(model, LanguageModel): + logger.error( + f"Model type mismatch: Expected LanguageModel but got {type(model).__name__}. " + f"Selection reason: {selection_reason}. " + f"model_id={model_id}, default_type={default_type}." + ) + raise ConfigurationError( + f"Model is not a LanguageModel: {model}. " + f"Please check that the model configured for '{default_type}' is a language model, not an embedding or speech model." + ) + + return model.to_langchain() diff --git a/open_notebook/config.py b/open_notebook/config.py new file mode 100644 index 0000000000000000000000000000000000000000..2db8f74833e9f8cf88ac399543cbf9665a66b0bf --- /dev/null +++ b/open_notebook/config.py @@ -0,0 +1,20 @@ +import os + +# ROOT DATA FOLDER +DATA_FOLDER = "./data" + +# LANGGRAPH CHECKPOINT FILE +sqlite_folder = f"{DATA_FOLDER}/sqlite-db" +os.makedirs(sqlite_folder, exist_ok=True) +LANGGRAPH_CHECKPOINT_FILE = f"{sqlite_folder}/checkpoints.sqlite" + +# UPLOADS FOLDER +UPLOADS_FOLDER = f"{DATA_FOLDER}/uploads" +os.makedirs(UPLOADS_FOLDER, exist_ok=True) + +# TIKTOKEN CACHE FOLDER +# Reads TIKTOKEN_CACHE_DIR from the environment so Docker can redirect the cache +# to a path outside /data/ (which is typically volume-mounted and would hide the +# pre-baked encoding baked into the image at build time). +TIKTOKEN_CACHE_DIR = os.environ.get("TIKTOKEN_CACHE_DIR", "").strip() or f"{DATA_FOLDER}/tiktoken-cache" +os.makedirs(TIKTOKEN_CACHE_DIR, exist_ok=True) diff --git a/open_notebook/database/CLAUDE.md b/open_notebook/database/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..bf8ecb7d7307c235399b0b89ab0f1976c1f41b88 --- /dev/null +++ b/open_notebook/database/CLAUDE.md @@ -0,0 +1,124 @@ +# Database Module + +SurrealDB abstraction layer providing repository pattern for CRUD operations and async migration management. + +## Purpose + +Encapsulates all database interactions: connection pooling, async CRUD operations, relationship management, and schema migrations. Provides clean interface for domain models and API endpoints to interact with SurrealDB without direct query knowledge. + +## Architecture Overview + +Two-tier system: +1. **Repository Layer** (repository.py): Raw async CRUD operations on SurrealDB via AsyncSurreal client +2. **Migration Layer** (async_migrate.py): Schema versioning and migration execution + +Both leverage connection context manager for lifecycle management and automatic cleanup. + +## Component Catalog + +### repository.py + +**Connection Management** +- `get_database_url()`: Resolves `SURREAL_URL` or constructs from `SURREAL_ADDRESS`/`SURREAL_PORT` (backward compatible) +- `get_database_password()`: Falls back from `SURREAL_PASSWORD` to legacy `SURREAL_PASS` env var +- `db_connection()`: Async context manager handling sign-in, namespace/database selection, and cleanup + - Opens AsyncSurreal, authenticates, selects namespace/database, yields connection, closes on exit + +**Query Operations** +- `repo_query(query_str, vars)`: Execute raw SurrealQL with parameter substitution; returns list of dicts +- `repo_create(table, data)`: Insert record; auto-adds `created`/`updated` timestamps; removes any existing `id` field +- `repo_insert(table, data_list, ignore_duplicates)`: Bulk insert multiple records; optionally ignores "already contains" errors +- `repo_upsert(table, id, data, add_timestamp)`: MERGE operation for create-or-update; optionally adds `updated` timestamp +- `repo_update(table, id, data)`: Update existing record by table+id or full record_id; auto-adds `updated`, parses ISO dates +- `repo_delete(record_id)`: Delete record by RecordID +- `repo_relate(source, relationship, target, data)`: Create graph relationship; optional relationship data + +**Utilities** +- `parse_record_ids(obj)`: Recursively converts SurrealDB RecordID objects to strings (deep tree traversal) +- `ensure_record_id(value)`: Coerces string or RecordID to RecordID type + +### async_migrate.py + +**Migration Classes** +- `AsyncMigration`: Single migration wrapper + - `from_file(path)`: Load .surrealql file; strips comments and whitespace + - `run(bump)`: Execute SQL; call bump_version() on success (bump=True) or lower_version() (bump=False) + +- `AsyncMigrationRunner`: Sequences multiple migrations + - `run_all()`: Execute pending migrations from current_version to end + - `run_one_up()`: Run next migration + - `run_one_down()`: Rollback latest migration + +- `AsyncMigrationManager`: Main orchestrator + - Loads 14 up migrations + 14 down migrations (hard-coded in __init__; migrations 11-12 add credential system, 13 adds model-credential link, 14 adds podcast model registry fields) + - `get_current_version()`: Query max version from _sbl_migrations table + - `needs_migration()`: Boolean check (current < total migrations available) + - `run_migration_up()`: Run all pending migrations with logging + +**Version Tracking** +- `get_latest_version()`: Query max version; returns 0 if _sbl_migrations table missing +- `get_all_versions()`: Fetch all migration records; returns empty list on error +- `bump_version()`: INSERT new entry into _sbl_migrations with version + applied_at timestamp +- `lower_version()`: DELETE latest migration record (rollback) + +### migrate.py + +**Backward Compatibility** +- `MigrationManager`: Sync wrapper around AsyncMigrationManager + - `get_current_version()`: Wraps async call with asyncio.run() + - `needs_migration` property: Checks if migration pending + - `run_migration_up()`: Execute migrations synchronously + +## Common Patterns + +- **Async-first design**: All operations async via AsyncSurreal; sync wrapper provided for legacy code +- **Connection per operation**: Each repo_* function opens/closes connection (no pooling); designed for serverless/stateless API +- **Auto-timestamping**: repo_create() and repo_update() auto-set `created`/`updated` fields +- **Error resilience**: RuntimeError for transaction conflicts (retriable, logged at DEBUG level); catches and re-raises other exceptions +- **RecordID polymorphism**: Functions accept string or RecordID; coerced to consistent type +- **Graceful degradation**: Migration queries catch exceptions and treat table-not-found as version 0 + +## Key Dependencies + +- `surrealdb`: AsyncSurreal client, RecordID type +- `loguru`: Logging with context (debug/error/success levels) +- Python stdlib: `os` (env vars), `datetime` (timestamps), `contextlib` (async context manager) + +## Important Quirks & Gotchas + +- **No connection pooling**: Each repo_* operation creates new connection; adequate for HTTP request-scoped operations but inefficient for bulk workloads +- **Hard-coded migration files**: AsyncMigrationManager lists migrations 1-14 explicitly; adding new migration requires code change (not auto-discovery) +- **Record ID format inconsistency**: repo_update() accepts both `table:id` format and full RecordID; path handling can be subtle +- **ISO date parsing**: repo_update() parses `created` field from string to datetime if present; assumes ISO format +- **Timestamp overwrite risk**: repo_create() always sets new timestamps; can't preserve original created time on reimport +- **Transaction conflict handling**: RuntimeError from transaction conflicts logged at DEBUG level without stack trace (prevents log spam during concurrent operations) +- **Graceful null returns**: get_all_versions() returns [] on table missing; allows migration system to bootstrap cleanly + +## How to Extend + +1. **Add new CRUD operation**: Follow repo_* pattern (open connection, execute query, handle errors, close) +2. **Add migration**: Create migration file in `/migrations/N.surrealql` and `/migrations/N_down.surrealql`; update AsyncMigrationManager to load new files +3. **Change timestamp behavior**: Modify repo_create()/repo_update() to not auto-set `updated` field if caller-provided +4. **Implement connection pooling**: Replace db_connection context manager with pool.acquire() pattern (for high-throughput scenarios) + +## Integration Points + +- **API startup** (api/main.py): FastAPI lifespan handler calls AsyncMigrationManager.run_migration_up() on server start +- **Domain models** (domain/*.py): All models call repo_* functions for persistence +- **Commands** (commands/*.py): Background jobs use repo_* for state updates +- **Streamlit UI** (pages/*.py): Deprecated migration check; relies on API to run migrations + +## Usage Example + +```python +from open_notebook.database.repository import repo_create, repo_query, repo_update + +# Create +record = await repo_create("notebooks", {"title": "Research"}) + +# Query +results = await repo_query("SELECT * FROM notebooks WHERE title = $title", {"title": "Research"}) + +# Update +await repo_update("notebooks", record["id"], {"title": "Updated Research"}) +``` diff --git a/open_notebook/database/async_migrate.py b/open_notebook/database/async_migrate.py new file mode 100644 index 0000000000000000000000000000000000000000..ef8cac8df49166c224270e5a8d8fc4ba75133cd8 --- /dev/null +++ b/open_notebook/database/async_migrate.py @@ -0,0 +1,238 @@ +""" +Async migration system for SurrealDB using the official Python client. +Based on patterns from sblpy migration system. +""" + +from typing import List + +from loguru import logger + +from .repository import db_connection, repo_query + + +class AsyncMigration: + """ + Handles individual migration operations with async support. + """ + + def __init__(self, sql: str) -> None: + """Initialize migration with SQL content.""" + self.sql = sql + + @classmethod + def from_file(cls, file_path: str) -> "AsyncMigration": + """Create migration from SQL file.""" + with open(file_path, "r", encoding="utf-8") as file: + raw_content = file.read() + # Clean up SQL content + lines = [] + for line in raw_content.split("\n"): + line = line.strip() + if line and not line.startswith("--"): + lines.append(line) + sql = " ".join(lines) + return cls(sql) + + async def run(self, bump: bool = True) -> None: + """Run the migration.""" + try: + async with db_connection() as connection: + await connection.query(self.sql) + + if bump: + await bump_version() + else: + await lower_version() + + except Exception as e: + logger.error(f"Migration failed: {str(e)}") + raise + + +class AsyncMigrationRunner: + """ + Handles running multiple migrations in sequence. + """ + + def __init__( + self, + up_migrations: List[AsyncMigration], + down_migrations: List[AsyncMigration], + ) -> None: + """Initialize runner with migration lists.""" + self.up_migrations = up_migrations + self.down_migrations = down_migrations + + async def run_all(self) -> None: + """Run all pending up migrations.""" + current_version = await get_latest_version() + + for i in range(current_version, len(self.up_migrations)): + logger.info(f"Running migration {i + 1}") + await self.up_migrations[i].run(bump=True) + + async def run_one_up(self) -> None: + """Run one up migration.""" + current_version = await get_latest_version() + + if current_version < len(self.up_migrations): + logger.info(f"Running migration {current_version + 1}") + await self.up_migrations[current_version].run(bump=True) + + async def run_one_down(self) -> None: + """Run one down migration.""" + current_version = await get_latest_version() + + if current_version > 0: + logger.info(f"Rolling back migration {current_version}") + await self.down_migrations[current_version - 1].run(bump=False) + + +class AsyncMigrationManager: + """ + Main migration manager with async support. + """ + + def __init__(self): + """Initialize migration manager.""" + self.up_migrations = [ + AsyncMigration.from_file("open_notebook/database/migrations/1.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/2.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/3.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/4.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/5.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/6.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/7.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/8.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/9.surrealql"), + AsyncMigration.from_file("open_notebook/database/migrations/10.surrealql"), + AsyncMigration.from_file( + "open_notebook/database/migrations/11.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/12.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/13.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/14.surrealql" + ), + ] + self.down_migrations = [ + AsyncMigration.from_file( + "open_notebook/database/migrations/1_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/2_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/3_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/4_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/5_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/6_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/7_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/8_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/9_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/10_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/11_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/12_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/13_down.surrealql" + ), + AsyncMigration.from_file( + "open_notebook/database/migrations/14_down.surrealql" + ), + ] + self.runner = AsyncMigrationRunner( + up_migrations=self.up_migrations, + down_migrations=self.down_migrations, + ) + + async def get_current_version(self) -> int: + """Get current database version.""" + return await get_latest_version() + + async def needs_migration(self) -> bool: + """Check if migration is needed.""" + current_version = await self.get_current_version() + return current_version < len(self.up_migrations) + + async def run_migration_up(self): + """Run all pending migrations.""" + current_version = await self.get_current_version() + logger.info(f"Current version before migration: {current_version}") + + if await self.needs_migration(): + try: + await self.runner.run_all() + new_version = await self.get_current_version() + logger.info(f"Migration successful. New version: {new_version}") + except Exception as e: + logger.error(f"Migration failed: {str(e)}") + raise + else: + logger.info("Database is already at the latest version") + + +# Database version management functions +async def get_latest_version() -> int: + """Get the latest version from the migrations table.""" + try: + versions = await get_all_versions() + if not versions: + return 0 + return max(version["version"] for version in versions) + except Exception: + # If migrations table doesn't exist, we're at version 0 + return 0 + + +async def get_all_versions() -> List[dict]: + """Get all versions from the migrations table.""" + try: + result = await repo_query("SELECT * FROM _sbl_migrations ORDER BY version;") + return result + except Exception: + # If table doesn't exist, return empty list + return [] + + +async def bump_version() -> None: + """Bump the version by adding a new entry to migrations table.""" + current_version = await get_latest_version() + new_version = current_version + 1 + + await repo_query( + "CREATE type::thing('_sbl_migrations', $version) SET version = $version, applied_at = time::now();", + {"version": new_version}, + ) + + +async def lower_version() -> None: + """Lower the version by removing the latest entry from migrations table.""" + current_version = await get_latest_version() + if current_version > 0: + await repo_query( + "DELETE type::thing('_sbl_migrations', $version);", + {"version": current_version}, + ) diff --git a/open_notebook/database/migrate.py b/open_notebook/database/migrate.py new file mode 100644 index 0000000000000000000000000000000000000000..7573ba0ca5d73e1dccc8f3fbf1d36f2b9816d533 --- /dev/null +++ b/open_notebook/database/migrate.py @@ -0,0 +1,26 @@ +import asyncio + +from .async_migrate import AsyncMigrationManager + + +class MigrationManager: + """ + Synchronous wrapper around AsyncMigrationManager for backward compatibility. + """ + + def __init__(self): + """Initialize with async migration manager.""" + self._async_manager = AsyncMigrationManager() + + def get_current_version(self) -> int: + """Get current database version (sync wrapper).""" + return asyncio.run(self._async_manager.get_current_version()) + + @property + def needs_migration(self) -> bool: + """Check if migration is needed (sync wrapper).""" + return asyncio.run(self._async_manager.needs_migration()) + + def run_migration_up(self): + """Run migrations (sync wrapper).""" + asyncio.run(self._async_manager.run_migration_up()) diff --git a/open_notebook/database/migrations/1.surrealql b/open_notebook/database/migrations/1.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..8ca52ca80bb4c5642c7d86b25a570960eea024f7 --- /dev/null +++ b/open_notebook/database/migrations/1.surrealql @@ -0,0 +1,178 @@ + +DEFINE TABLE IF NOT EXISTS source SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS + asset + ON TABLE source + FLEXIBLE TYPE option; + +DEFINE FIELD IF NOT EXISTS title ON TABLE source TYPE option; +DEFINE FIELD IF NOT EXISTS topics ON TABLE source TYPE option>; +DEFINE FIELD IF NOT EXISTS full_text ON TABLE source TYPE option; + +DEFINE FIELD IF NOT EXISTS created ON source DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON source DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS source_embedding SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS source ON TABLE source_embedding TYPE record; +DEFINE FIELD IF NOT EXISTS order ON TABLE source_embedding TYPE int; +DEFINE FIELD IF NOT EXISTS content ON TABLE source_embedding TYPE string; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_embedding TYPE array; + +DEFINE TABLE IF NOT EXISTS source_insight SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS source ON TABLE source_insight TYPE record; +DEFINE FIELD IF NOT EXISTS insight_type ON TABLE source_insight TYPE string; +DEFINE FIELD IF NOT EXISTS content ON TABLE source_insight TYPE string; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_insight TYPE array; + + +DEFINE EVENT IF NOT EXISTS source_delete ON TABLE source WHEN ($after == NONE) THEN { + delete source_embedding where source == $before.id; + delete source_insight where source == $before.id; +}; + +DEFINE TABLE IF NOT EXISTS note SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS title ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS summary ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS content ON TABLE note TYPE option; +DEFINE FIELD IF NOT EXISTS embedding ON TABLE note TYPE array; + +DEFINE FIELD IF NOT EXISTS created ON note DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON note DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS notebook SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS name ON TABLE notebook TYPE option; +DEFINE FIELD IF NOT EXISTS description ON TABLE notebook TYPE option; +DEFINE FIELD IF NOT EXISTS archived ON TABLE notebook TYPE option DEFAULT False; + + +DEFINE FIELD IF NOT EXISTS created ON notebook DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON notebook DEFAULT time::now() VALUE time::now(); + +DEFINE TABLE IF NOT EXISTS reference +TYPE RELATION +FROM source TO notebook; + +DEFINE TABLE IF NOT EXISTS artifact +TYPE RELATION +FROM note TO notebook; + +DEFINE TABLE IF NOT EXISTS podcast_config SCHEMALESS; + +-- entender o analyzer +DEFINE ANALYZER IF NOT EXISTS my_analyzer TOKENIZERS blank,class,camel,punct FILTERS snowball(english), lowercase; + +DEFINE INDEX IF NOT EXISTS idx_source_title ON TABLE source COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_full_text ON TABLE source COLUMNS full_text SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_embed_chunk ON TABLE source_embedding COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_source_insight ON TABLE source_insight COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_note ON TABLE note COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; +DEFINE INDEX IF NOT EXISTS idx_note_title ON TABLE note COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS; + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results + group by item_id ORDER BY relevance DESC LIMIT $match_count); + + +}; + + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding LIMIT $match_count)} + ELSE { [] }; + + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight LIMIT $match_count)} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM note LIMIT $match_count)} + + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_insight_search); + + let $note_results = $note_content_search; + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results + group by item_id ORDER BY similarity DESC LIMIT $match_count); + + +}; + +IF array::len(select * from open_notebook:default_models) == 0 THEN + CREATE open_notebook:default_models SET + default_chat_model= "" +END; diff --git a/open_notebook/database/migrations/10.surrealql b/open_notebook/database/migrations/10.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..a6a6f4e6488cdbbbd533050cce23bc15b4c52e43 --- /dev/null +++ b/open_notebook/database/migrations/10.surrealql @@ -0,0 +1,13 @@ +-- Migration 10: Add indexes for source_insight and source_embedding source field +-- These indexes significantly improve performance of source listing queries +-- that count insights and check embedding existence per source + +DEFINE INDEX IF NOT EXISTS idx_source_insight_source ON source_insight FIELDS source CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_source_embedding_source ON source_embedding FIELDS source CONCURRENTLY; + +DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE option>; +DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE option>; + +-- delete orphan records +DELETE from source_embedding WHERE source.id=NONE; +DELETE from source_insight WHERE source.id=NONE; diff --git a/open_notebook/database/migrations/10_down.surrealql b/open_notebook/database/migrations/10_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..e23b979e566a78bb3bd277e8208f37eaa07e8d71 --- /dev/null +++ b/open_notebook/database/migrations/10_down.surrealql @@ -0,0 +1,4 @@ +-- Rollback Migration 10: Remove source field indexes + +REMOVE INDEX IF EXISTS idx_source_insight_source ON TABLE source_insight; +REMOVE INDEX IF EXISTS idx_source_embedding_source ON TABLE source_embedding; diff --git a/open_notebook/database/migrations/11.surrealql b/open_notebook/database/migrations/11.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..f5e195df80f9b6320fc73288809fa4dadfcaa429 --- /dev/null +++ b/open_notebook/database/migrations/11.surrealql @@ -0,0 +1,10 @@ +-- Migration 11: Create provider configuration singleton record +-- This record stores multiple API key configurations per provider +-- The data is managed by the ProviderConfig RecordModel class + +-- Create the provider configs singleton record for multi-config support +-- This record stores multiple API key configurations per provider +-- The data is managed by the ProviderConfig RecordModel class +UPSERT open_notebook:provider_configs CONTENT { + credentials: {} +}; diff --git a/open_notebook/database/migrations/11_down.surrealql b/open_notebook/database/migrations/11_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..4e75201fe0b5a34e5cbd6ff4b9c13c4c5b07a5df --- /dev/null +++ b/open_notebook/database/migrations/11_down.surrealql @@ -0,0 +1,4 @@ +-- Rollback Migration 11: Remove provider configuration records + +-- Remove provider configs singleton (if exists) +DELETE open_notebook:provider_configs; diff --git a/open_notebook/database/migrations/12.surrealql b/open_notebook/database/migrations/12.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..c5aca94fcac805480d55f42c127d271ef05350dd --- /dev/null +++ b/open_notebook/database/migrations/12.surrealql @@ -0,0 +1,29 @@ +-- Migration 12: Create credential table and add credential link to model table +-- Individual credential records replace the ProviderConfig singleton +-- Each credential stores API key and provider-specific configuration + + +DEFINE TABLE credential SCHEMAFULL; +DEFINE FIELD name ON credential TYPE string; +DEFINE FIELD provider ON credential TYPE string; +DEFINE FIELD modalities ON credential TYPE array DEFAULT []; +DEFINE FIELD modalities.* ON credential TYPE string; +DEFINE FIELD api_key ON credential TYPE option; +DEFINE FIELD base_url ON credential TYPE option; +DEFINE FIELD endpoint ON credential TYPE option; +DEFINE FIELD api_version ON credential TYPE option; +DEFINE FIELD endpoint_llm ON credential TYPE option; +DEFINE FIELD endpoint_embedding ON credential TYPE option; +DEFINE FIELD endpoint_stt ON credential TYPE option; +DEFINE FIELD endpoint_tts ON credential TYPE option; +DEFINE FIELD project ON credential TYPE option; +DEFINE FIELD location ON credential TYPE option; +DEFINE FIELD credentials_path ON credential TYPE option; +DEFINE FIELD created ON credential TYPE option DEFAULT time::now(); +DEFINE FIELD updated ON credential TYPE option DEFAULT time::now(); + +-- Index for fast provider lookups +DEFINE INDEX idx_credential_provider ON credential FIELDS provider; + +-- Add optional credential link to model table +DEFINE FIELD credential ON model TYPE option>; diff --git a/open_notebook/database/migrations/12_down.surrealql b/open_notebook/database/migrations/12_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..c16b28b90f938312966d54c20e87cc23e5951225 --- /dev/null +++ b/open_notebook/database/migrations/12_down.surrealql @@ -0,0 +1,5 @@ +-- Rollback Migration 12: Remove credential table and credential field from model + +REMOVE FIELD credential ON TABLE model; +REMOVE INDEX idx_credential_provider ON credential; +REMOVE TABLE credential; diff --git a/open_notebook/database/migrations/13.surrealql b/open_notebook/database/migrations/13.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..fe35f009f80b73c21181125d5822ded95cbce7b0 --- /dev/null +++ b/open_notebook/database/migrations/13.surrealql @@ -0,0 +1,3 @@ + +DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE option>; +DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE option>; diff --git a/open_notebook/database/migrations/13_down.surrealql b/open_notebook/database/migrations/13_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..529d5283a442714c6bb7b8e2b1f4b9869465565c --- /dev/null +++ b/open_notebook/database/migrations/13_down.surrealql @@ -0,0 +1,2 @@ +DEFINE FIELD OVERWRITE embedding ON TABLE source_insight TYPE array; +DEFINE FIELD OVERWRITE embedding ON TABLE note TYPE array; \ No newline at end of file diff --git a/open_notebook/database/migrations/14.surrealql b/open_notebook/database/migrations/14.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..91bdcb8c47f13a1e57d11bc642d7b4dfdfdb38ba --- /dev/null +++ b/open_notebook/database/migrations/14.surrealql @@ -0,0 +1,27 @@ +-- Migration 14: Podcast profiles model registry integration +-- Adds record references to replace loose provider/model strings +-- Adds language field to episode_profile +-- Adds per-speaker TTS override support + +-- EPISODE PROFILE +-- Legacy fields: make optional (app ignores, preserved for data migration) +DEFINE FIELD OVERWRITE outline_provider ON TABLE episode_profile TYPE option; +DEFINE FIELD OVERWRITE outline_model ON TABLE episode_profile TYPE option; +DEFINE FIELD OVERWRITE transcript_provider ON TABLE episode_profile TYPE option; +DEFINE FIELD OVERWRITE transcript_model ON TABLE episode_profile TYPE option; + +-- New fields: reference to Model registry +DEFINE FIELD IF NOT EXISTS outline_llm ON TABLE episode_profile TYPE option>; +DEFINE FIELD IF NOT EXISTS transcript_llm ON TABLE episode_profile TYPE option>; +DEFINE FIELD IF NOT EXISTS language ON TABLE episode_profile TYPE option; + +-- SPEAKER PROFILE +-- Legacy fields: make optional +DEFINE FIELD OVERWRITE tts_provider ON TABLE speaker_profile TYPE option; +DEFINE FIELD OVERWRITE tts_model ON TABLE speaker_profile TYPE option; + +-- New field: reference to Model registry (profile-level) +DEFINE FIELD IF NOT EXISTS voice_model ON TABLE speaker_profile TYPE option>; + +-- Per-speaker TTS override +DEFINE FIELD IF NOT EXISTS speakers.*.voice_model ON TABLE speaker_profile TYPE option>; diff --git a/open_notebook/database/migrations/14_down.surrealql b/open_notebook/database/migrations/14_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..99a3dc4edb7b383d4ba7cc9f419bce435b381feb --- /dev/null +++ b/open_notebook/database/migrations/14_down.surrealql @@ -0,0 +1,20 @@ +-- Migration 14 rollback: Remove model registry fields from podcast profiles + +-- Remove new fields from episode_profile +REMOVE FIELD IF EXISTS outline_llm ON TABLE episode_profile; +REMOVE FIELD IF EXISTS transcript_llm ON TABLE episode_profile; +REMOVE FIELD IF EXISTS language ON TABLE episode_profile; + +-- Restore episode_profile legacy fields as required strings +DEFINE FIELD OVERWRITE outline_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD OVERWRITE outline_model ON TABLE episode_profile TYPE string; +DEFINE FIELD OVERWRITE transcript_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD OVERWRITE transcript_model ON TABLE episode_profile TYPE string; + +-- Remove new fields from speaker_profile +REMOVE FIELD IF EXISTS voice_model ON TABLE speaker_profile; +REMOVE FIELD IF EXISTS speakers.*.voice_model ON TABLE speaker_profile; + +-- Restore speaker_profile legacy fields as required strings +DEFINE FIELD OVERWRITE tts_provider ON TABLE speaker_profile TYPE string; +DEFINE FIELD OVERWRITE tts_model ON TABLE speaker_profile TYPE string; diff --git a/open_notebook/database/migrations/1_down.surrealql b/open_notebook/database/migrations/1_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..e53791aac000cbe983a2846652bbfd06fdf58c72 --- /dev/null +++ b/open_notebook/database/migrations/1_down.surrealql @@ -0,0 +1,24 @@ +REMOVE TABLE IF EXISTS source; +REMOVE TABLE IF EXISTS source_embedding; +REMOVE TABLE IF EXISTS source_insight; +REMOVE TABLE IF EXISTS note; +REMOVE TABLE IF EXISTS notebook; +REMOVE TABLE IF EXISTS reference; +REMOVE TABLE IF EXISTS artifact; +REMOVE TABLE IF EXISTS podcast_config; + +REMOVE EVENT IF EXISTS source_delete ON TABLE source; + +REMOVE ANALYZER IF EXISTS my_analyzer; + +REMOVE INDEX IF EXISTS idx_source_title ON TABLE source; +REMOVE INDEX IF EXISTS idx_source_full_text ON TABLE source; +REMOVE INDEX IF EXISTS idx_source_embed_chunk ON TABLE source_embedding; +REMOVE INDEX IF EXISTS idx_source_insight ON TABLE source_insight; +REMOVE INDEX IF EXISTS idx_note ON TABLE note; +REMOVE INDEX IF EXISTS idx_note_title ON TABLE note; + +REMOVE FUNCTION IF EXISTS fn::text_search; +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DELETE open_notebook:default_models; diff --git a/open_notebook/database/migrations/2.surrealql b/open_notebook/database/migrations/2.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..66c10172169fd49a942766c224032a5c19f7dd46 --- /dev/null +++ b/open_notebook/database/migrations/2.surrealql @@ -0,0 +1 @@ +DEFINE FIELD IF NOT EXISTS note_type ON TABLE note TYPE option; diff --git a/open_notebook/database/migrations/2_down.surrealql b/open_notebook/database/migrations/2_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..d0213a0e285d106b1f62afe0a5bade9c68db7752 --- /dev/null +++ b/open_notebook/database/migrations/2_down.surrealql @@ -0,0 +1 @@ +REMOVE FIELD IF EXISTS note_type ON TABLE note; diff --git a/open_notebook/database/migrations/3.surrealql b/open_notebook/database/migrations/3.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..f2a067f84171f9de24a310ed66fe78d0ca34d44e --- /dev/null +++ b/open_notebook/database/migrations/3.surrealql @@ -0,0 +1,145 @@ + +DEFINE TABLE IF NOT EXISTS chat_session SCHEMALESS; + +DEFINE TABLE IF NOT EXISTS refers_to +TYPE RELATION +FROM chat_session TO notebook; + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN ( + SELECT + id, title, content, parent_id, + math::max(similarity) as similarity + FROM $all_results + GROUP BY id + ORDER BY similarity DESC + LIMIT $match_count + ); +}; + + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results + where id is not None +group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/open_notebook/database/migrations/3_down.surrealql b/open_notebook/database/migrations/3_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..b8438e0a9448050b6de11821167f33e6c48eb3db --- /dev/null +++ b/open_notebook/database/migrations/3_down.surrealql @@ -0,0 +1,110 @@ +REMOVE TABLE IF EXISTS chat_session; + +REMOVE TABLE IF EXISTS refers_to; + + +REMOVE FUNCTION fn::vector_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding LIMIT $match_count)} + ELSE { [] }; + + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight LIMIT $match_count)} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity + FROM note LIMIT $match_count)} + + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_insight_search); + + let $note_results = $note_content_search; + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results + group by item_id ORDER BY similarity DESC LIMIT $match_count); + + +}; + +REMOVE FUNCTION fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT source as item_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id as item_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY item_id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results + group by item_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/open_notebook/database/migrations/4.surrealql b/open_notebook/database/migrations/4.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..77440870f8f61fb3add224c77c9fd94c587472ca --- /dev/null +++ b/open_notebook/database/migrations/4.surrealql @@ -0,0 +1,134 @@ + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + (source.title OR '') as title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (select id, parent_id, title, math::max(relevance) as relevance + from $final_results where id is not None + group by id, parent_id, title ORDER BY relevance DESC LIMIT $match_count); + +}; + + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/open_notebook/database/migrations/4_down.surrealql b/open_notebook/database/migrations/4_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..67acbd7166cc985bd68d4d6dbd6027550f93bbb2 --- /dev/null +++ b/open_notebook/database/migrations/4_down.surrealql @@ -0,0 +1,139 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN ( + SELECT + id, title, content, parent_id, + math::max(similarity) as similarity + FROM $all_results + GROUP BY id + ORDER BY similarity DESC + LIMIT $match_count + ); +}; + + +REMOVE FUNCTION IF EXISTS fn::text_search; + + +DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) { + + let $source_title_search = + IF $sources {( + SELECT id, title, + search::highlight('`', '`', 1) as content, + id as parent_id, + math::max(search::score(1)) AS relevance + FROM source + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_embedding_search = + IF $sources {( + SELECT id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_embedding + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_full_search = + IF $sources {( + SELECT source.id as id, source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source + WHERE full_text @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT id, insight_type + " - " + source.title as title, search::highlight('`', '`', 1) as content, source.id as parent_id, math::max(search::score(1)) AS relevance + FROM source_insight + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_title_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE title @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $note_content_search = + IF $show_notes {( + SELECT id, title, search::highlight('`', '`', 1) as content, id as parent_id, math::max(search::score(1)) AS relevance + FROM note + WHERE content @1@ $query_text + GROUP BY id)} + ELSE { [] }; + + let $source_chunk_results = array::union($source_embedding_search, $source_full_search); + + let $source_asset_results = array::union($source_title_search, $source_insight_search); + + let $source_results = array::union($source_chunk_results, $source_asset_results ); + let $note_results = array::union($note_title_search, $note_content_search ); + let $final_results = array::union($source_results, $note_results ); + + RETURN (SELECT id, title, content, parent_id, math::max(relevance) as relevance from $final_results + where id is not None +group by id, title, content, parent_id ORDER BY relevance DESC LIMIT $match_count); + + +}; diff --git a/open_notebook/database/migrations/5.surrealql b/open_notebook/database/migrations/5.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..3fd2a51522e7c5ea187ba12b40ac06268ec30105 --- /dev/null +++ b/open_notebook/database/migrations/5.surrealql @@ -0,0 +1,169 @@ + +-- remove old transformation defaults + +DELETE open_notebook:default_transformations; + +-- set up the default transformations + +DEFINE TABLE IF NOT EXISTS transformation SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS name ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS title ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS prompt ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS apply_default ON TABLE transformation TYPE bool DEFAULT False; +DEFINE FIELD IF NOT EXISTS created ON transformation DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON transformation DEFAULT time::now() VALUE time::now(); + + +insert into transformation [ + { + name: "Analyze Paper", + title: "Paper Analysis", + description: "Analyses a technical/scientific paper", + prompt:"# IDENTITY and PURPOSE + +You are an insightful and analytical reader of academic papers, extracting the key components, significance, and broader implications. Your focus is to uncover the core contributions, practical applications, methodological strengths or weaknesses, and any surprising findings. You are especially attuned to the clarity of arguments, the relevance to existing literature, and potential impacts on both the specific field and broader contexts. + +# STEPS + +1. **READ AND UNDERSTAND THE PAPER**: Thoroughly read the paper, identifying its main focus, arguments, methods, results, and conclusions. + +2. **IDENTIFY CORE ELEMENTS**: + - **Purpose**: What is the main goal or research question? + - **Contribution**: What new knowledge or innovation does this paper bring to the field? + - **Methods**: What methods are used, and are they novel or particularly effective? + - **Key Findings**: What are the most critical results, and why do they matter? + - **Limitations**: Are there any notable limitations or areas for further research? + +3. **SYNTHESIZE THE MAIN POINTS**: + - Extract the key elements and organize them into insightful observations. + - Highlight the broader impact and potential applications. + - Note any aspects that challenge established views or introduce new questions. + +# OUTPUT INSTRUCTIONS + +- Structure the output as follows: + - **PURPOSE**: A concise summary of the main research question or goal (1-2 sentences). + - **CONTRIBUTION**: A bullet list of 2-3 points that describe what the paper adds to the field. + - **KEY FINDINGS**: A bullet list of 2-3 points summarizing the critical outcomes of the study. + - **IMPLICATIONS**: A bullet list of 2-3 points discussing the significance or potential impact of the findings on the field or broader context. + - **LIMITATIONS**: A bullet list of 1-2 points identifying notable limitations or areas for future work. + +- **Bullet Points** should be between 15-20 words. +- Avoid starting each bullet point with the same word to maintain variety. +- Use clear and concise language that conveys the key ideas effectively. +- Do not include warnings, disclaimers, or personal opinions. +- Output only the requested sections with their respective labels.", + apply_default: False + }, + { + name: "Key Insights", + title: "Key Insights", + description: "Extracts important insights and actionable items", + prompt:"# IDENTITY and PURPOSE + +You extract surprising, powerful, and interesting insights from text content. You are interested in insights related to the purpose and meaning of life, human flourishing, the role of technology in the future of humanity, artificial intelligence and its affect on humans, memes, learning, reading, books, continuous improvement, and similar topics. +You create 15 word bullet points that capture the most important insights from the input. +Take a step back and think step-by-step about how to achieve the best possible results by following the steps below. + +# STEPS + +- Extract 20 to 50 of the most surprising, insightful, and/or interesting ideas from the input in a section called IDEAS, and write them on a virtual whiteboard in your mind using 15 word bullets. If there are less than 50 then collect all of them. Make sure you extract at least 20. + +- From those IDEAS, extract the most powerful and insightful of them and write them in a section called INSIGHTS. Make sure you extract at least 10 and up to 25. + +# OUTPUT INSTRUCTIONS + +- INSIGHTS are essentially higher-level IDEAS that are more abstracted and wise. +- Output the INSIGHTS section only. +- Each bullet should be about 15 words in length. +- Do not give warnings or notes; only output the requested sections. +- You use bulleted lists for output, not numbered lists. +- Do not start items with the same opening words. +- Ensure you follow ALL these instructions when creating your output. +", + apply_default: False + }, +{ + name: "Dense Summary", + title: "Dense Summary", + description: "Creates a rich, deep summary of the content", + prompt:"# MISSION +You are a Sparse Priming Representation (SPR) writer. An SPR is a particular kind of use of language for advanced NLP, NLU, and NLG tasks, particularly useful for the latest generation of Large Language Models (LLMs). You will be given information by the USER which you are to render as an SPR. + +# THEORY +LLMs are a kind of deep neural network. They have been demonstrated to embed knowledge, abilities, and concepts, ranging from reasoning to planning, and even to theory of mind. These are called latent abilities and latent content, collectively referred to as latent space. The latent space of an LLM can be activated with the correct series of words as inputs, which will create a useful internal state of the neural network. This is not unlike how the right shorthand cues can prime a human mind to think in a certain way. Like human minds, LLMs are associative, meaning you only need to use the correct associations to 'prime' another model to think in the same way. + +# METHODOLOGY +Render the input as a distilled list of succinct statements, assertions, associations, concepts, analogies, and metaphors. The idea is to capture as much, conceptually, as possible but with as few words as possible. Write it in a way that makes sense to you, as the future audience will be another language model, not a human. Use complete sentences. +", + apply_default: True + }, +{ + name: "Reflections", + title: "Reflection Questions", + description: "Generates reflection questions from the document to help explore it further", + prompt:"# IDENTITY and PURPOSE + +You extract deep, thought-provoking, and meaningful reflections from text content. You are especially focused on themes related to the human experience, such as the purpose of life, personal growth, the intersection of technology and humanity, artificial intelligence's societal impact, human potential, collective evolution, and transformative learning. Your reflections aim to provoke new ways of thinking, challenge assumptions, and provide a thoughtful synthesis of the content. + +# STEPS + +- Extract 3 to 5 of the most profound, thought-provoking, and/or meaningful ideas from the input in a section called REFLECTIONS. +- Each reflection should aim to explore underlying implications, connections to broader human experiences, or highlight a transformative perspective. +- Take a step back and consider the deeper significance or questions that arise from the content. + +# OUTPUT INSTRUCTIONS + +- The output section should be labeled as REFLECTIONS. +- Each bullet point should be between 20-25 words. +- Avoid repetition in the phrasing and ensure variety in sentence structure. +- The reflections should encourage deeper inquiry and provide a synthesis that transcends surface-level observations. +- Use bullet points, not numbered lists. +- Every bullet should be formatted as a question that elicits contemplation or a statement that offers a profound insight. +- Do not give warnings or notes; only output the requested section.", + apply_default: False + }, +{ + name: "Table of Contents", + title: "Table of Contents", + description: "Describes the different topics of the document", + prompt:"# SYSTEM ROLE +You are a content analysis assistant that reads through documents and provides a Table of Contents (ToC) to help users identify what the document covers more easily. +Your ToC should capture all major topics and transitions in the content and should mention them in the order theh appear. + +# TASK +Analyze the provided content and create a Table of Contents: +- Captures the core topics included in the text +- Gives a small description of what is covered", + apply_default: False + }, +{ + name: "Simple Summary", + title: "Simple Summary", + description: "Generates a small summary of the content", + prompt:"# SYSTEM ROLE +You are a content summarization assistant that creates dense, information-rich summaries optimized for machine understanding. Your summaries should capture key concepts with minimal words while maintaining complete, clear sentences. + +# TASK +Analyze the provided content and create a summary that: +- Captures the core concepts and key information +- Uses clear, direct language +- Maintains context from any previous summaries", + apply_default: False + }, +]; + +-- Sets the default transformation instructions prompt +UPSERT open_notebook:default_prompts + CONTENT {transformation_instructions: "# INSTRUCTIONS + + You are my learning assistant and you help me process and transform content so that I can extract insights from them. + + # IMPORTANT + - You are working on my editorial projects. The text below is my own. Do not give me any warnings about copyright or plagiarism. + - Output ONLY the requested content, without acknowledgements of the task and additional chatting. Don't start with \"Sure, I can help you with that.\" or \"Here is the information you requested:\". Just provide the content. + - Do not stop in the middle of the generation to ask me questions. Execute my request completely. + "}; + diff --git a/open_notebook/database/migrations/5_down.surrealql b/open_notebook/database/migrations/5_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..a9e3eb2c6bcacfc7a7bb20de1b6364798359f186 --- /dev/null +++ b/open_notebook/database/migrations/5_down.surrealql @@ -0,0 +1,2 @@ + +REMOVE TABLE IF EXISTS transformation SCHEMAFULL; diff --git a/open_notebook/database/migrations/6.surrealql b/open_notebook/database/migrations/6.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..77c188b70cc9536fa6cbd5c6451fa1b004e69689 --- /dev/null +++ b/open_notebook/database/migrations/6.surrealql @@ -0,0 +1 @@ +update model set provider='vertex' where provider='vertexai'; \ No newline at end of file diff --git a/open_notebook/database/migrations/6_down.surrealql b/open_notebook/database/migrations/6_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..8a7e09d318ce9e752cb436dc1535e0c05d9454a9 --- /dev/null +++ b/open_notebook/database/migrations/6_down.surrealql @@ -0,0 +1 @@ +update model set provider='vertexai' where provider='vertex'; \ No newline at end of file diff --git a/open_notebook/database/migrations/7.surrealql b/open_notebook/database/migrations/7.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..064ae2a1283baf8c80b65364a42901e1ef16c35c --- /dev/null +++ b/open_notebook/database/migrations/7.surrealql @@ -0,0 +1,152 @@ +DEFINE TABLE IF NOT EXISTS episode_profile SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE episode_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speaker_config ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS outline_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS outline_model ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS transcript_provider ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS transcript_model ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS default_briefing ON TABLE episode_profile TYPE string; +DEFINE FIELD IF NOT EXISTS num_segments ON TABLE episode_profile TYPE int DEFAULT 5; +DEFINE FIELD IF NOT EXISTS created ON TABLE episode_profile TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON TABLE episode_profile TYPE datetime DEFAULT time::now(); + +-- Create Speaker Profile table +remove table speaker_profile; +DEFINE TABLE IF NOT EXISTS speaker_profile SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS name ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS tts_provider ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS tts_model ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS speakers ON TABLE speaker_profile TYPE array; +DEFINE FIELD IF NOT EXISTS speakers.*.name ON TABLE speaker_profile TYPE string; +DEFINE FIELD IF NOT EXISTS speakers.*.voice_id ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speakers.*.backstory ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS speakers.*.personality ON TABLE speaker_profile TYPE option; +DEFINE FIELD IF NOT EXISTS created ON TABLE speaker_profile TYPE datetime DEFAULT time::now(); +DEFINE FIELD IF NOT EXISTS updated ON TABLE speaker_profile TYPE datetime DEFAULT time::now(); + + +-- Enhance PodcastEpisode table +DEFINE TABLE IF NOT EXISTS episode SCHEMAFULL; +DEFINE FIELD IF NOT EXISTS created ON episode DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON episode DEFAULT time::now() VALUE time::now(); +DEFINE FIELD IF NOT EXISTS name ON TABLE episode TYPE string; +DEFINE FIELD IF NOT EXISTS briefing ON TABLE episode TYPE option; +DEFINE FIELD IF NOT EXISTS episode_profile ON TABLE episode FLEXIBLE TYPE object; +DEFINE FIELD IF NOT EXISTS speaker_profile ON TABLE episode FLEXIBLE TYPE object; +DEFINE FIELD IF NOT EXISTS transcript ON TABLE episode FLEXIBLE TYPE option; +DEFINE FIELD IF NOT EXISTS outline ON TABLE episode FLEXIBLE TYPE option; +DEFINE FIELD IF NOT EXISTS command ON TABLE episode TYPE option>; +DEFINE FIELD IF NOT EXISTS content ON TABLE episode TYPE option; +DEFINE FIELD IF NOT EXISTS audio_file ON TABLE episode TYPE option; + +-- Create indexes for better performance +DEFINE INDEX IF NOT EXISTS idx_episode_profile_name ON TABLE episode_profile COLUMNS name UNIQUE CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_speaker_profile_name ON TABLE speaker_profile COLUMNS name UNIQUE CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_episode_profile ON TABLE episode COLUMNS episode_profile CONCURRENTLY; +DEFINE INDEX IF NOT EXISTS idx_episode_command ON TABLE episode COLUMNS command CONCURRENTLY; + + +--Sample data + +insert into episode_profile +[ + { + name: "tech_discussion", + description: "Technical discussion between 2 experts", + speaker_config: "tech_experts", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Create an engaging technical discussion about the provided content. Focus on practical insights, real-world applications, and detailed explanations that would interest developers and technical professionals.", + num_segments: 5 + }, + { + name: "solo_expert", + description: "Single expert explaining complex topics", + speaker_config: "solo_expert", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Create an educational explanation of the provided content. Break down complex concepts into digestible segments, use analogies and examples, and maintain an engaging teaching style.", + "num_segments":4 }, + { + name: "business_analysis", + description: "Business-focused analysis and discussion", + speaker_config: "business_panel", + outline_provider: "openai", + outline_model: "gpt-5-mini", + transcript_provider: "openai", + transcript_model: "gpt-5-mini", + default_briefing: "Analyze the provided content from a business perspective. Discuss market implications, strategic insights, competitive advantages, and actionable business intelligence.", + "num_segments":6 } + ]; + +insert into speaker_profile +[ + { + name: "tech_experts", + description: "Two technical experts for tech discussions", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Dr. Alex Chen", + voice_id: "nova", + backstory: "Senior AI researcher and former tech lead at major companies. Specializes in making complex technical concepts accessible.", + personality: "Analytical, clear communicator, asks probing questions to dig deeper into technical details" + }, + { + name: "Jamie Rodriguez", + voice_id: "alloy", + backstory: "Full-stack engineer and tech entrepreneur. Loves practical applications and real-world implementations.", + personality: "Enthusiastic, practical-minded, great at explaining implementation details and trade-offs" + } + ] + }, + { + name: "solo_expert", + description: "Single expert for educational content", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Professor Sarah Kim", + voice_id: "nova", + backstory: "Distinguished professor and researcher. Has a gift for making complex topics accessible to broad audiences.", + personality: "Patient teacher, uses analogies and examples, breaks down complex concepts step by step" + } + ] + }, + { + name: "business_panel", + description: "Business analysis panel with diverse perspectives", + tts_provider: "openai", + tts_model: "gpt-4o-mini-tts", + speakers: [ + { + name: "Marcus Thompson", + voice_id: "echo", + backstory: "Former McKinsey consultant, now startup advisor. Expert in strategic analysis and market dynamics.", + personality: "Strategic thinker, data-driven, excellent at identifying key insights and implications" + }, + { + name: "Elena Vasquez", + voice_id: "shimmer", + backstory: "Serial entrepreneur and investor. Focuses on practical implementation and execution.", + personality: "Action-oriented, pragmatic, brings startup experience and execution focus" + }, + { + name: "Johny Bing", + voice_id: "ash", + backstory: "Youtube celebrity and business mogul. Focuses on practical implementation and execution.", + personality: "Controversial, likes to question ideas and concepts. He brings a fresh perspective and always has a point to make." + } + ] + } + ]; + + diff --git a/open_notebook/database/migrations/7_down.surrealql b/open_notebook/database/migrations/7_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..de41849ed0b5d9016925823f7ae0fb72bcc89ae6 --- /dev/null +++ b/open_notebook/database/migrations/7_down.surrealql @@ -0,0 +1,3 @@ +REMOVE TABLE IF EXISTS episode_profile; +REMOVE TABLE IF EXISTS speaker_profile; +REMOVE TABLE IF EXISTS episode; diff --git a/open_notebook/database/migrations/8.surrealql b/open_notebook/database/migrations/8.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..f08a844d147fc017bbdd0b0eea3367c455e0c6b7 --- /dev/null +++ b/open_notebook/database/migrations/8.surrealql @@ -0,0 +1,12 @@ + +-- Migration 8: Support chat sessions for both notebooks and sources +-- This migration allows chat_session to refer to either a notebook or a source + +DEFINE TABLE OVERWRITE refers_to +TYPE RELATION +FROM chat_session TO notebook|source; + +-- Add model_override field to chat_session for per-session model selection +DEFINE FIELD model_override ON chat_session TYPE option; + +DEFINE FIELD command ON source TYPE option>; \ No newline at end of file diff --git a/open_notebook/database/migrations/8_down.surrealql b/open_notebook/database/migrations/8_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..6bb26ceb7811f6a0c331380a737f2a1d30906279 --- /dev/null +++ b/open_notebook/database/migrations/8_down.surrealql @@ -0,0 +1,11 @@ + +-- Rollback Migration 8: Revert to notebook-only chat sessions + +DEFINE TABLE OVERWRITE refers_to +TYPE RELATION +FROM chat_session TO notebook; + +-- Remove model_override field from chat_session +REMOVE FIELD model_override ON chat_session; + +REMOVE FIELD command ON source; diff --git a/open_notebook/database/migrations/9.surrealql b/open_notebook/database/migrations/9.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..582f41fe70f2b13231e3197c90625cef017ba802 --- /dev/null +++ b/open_notebook/database/migrations/9.surrealql @@ -0,0 +1,66 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE embedding != none and array::len(embedding)=array::len($query) AND + vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/open_notebook/database/migrations/9_down.surrealql b/open_notebook/database/migrations/9_down.surrealql new file mode 100644 index 0000000000000000000000000000000000000000..b383a524c2846273d349be14c99b276ff5c73e01 --- /dev/null +++ b/open_notebook/database/migrations/9_down.surrealql @@ -0,0 +1,63 @@ + +REMOVE FUNCTION IF EXISTS fn::vector_search; + +DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array, $match_count: int, $sources: bool, $show_notes: bool, $min_similarity: float) { + let $source_embedding_search = + IF $sources {( + SELECT + source.id as id, + source.title as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_embedding + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + let $source_insight_search = + IF $sources {( + SELECT + id, + insight_type + ' - ' + (source.title OR '') as title, + content, + source.id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM source_insight + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $note_content_search = + IF $show_notes {( + SELECT + id, + title, + content, + id as parent_id, + vector::similarity::cosine(embedding, $query) as similarity + FROM note + WHERE vector::similarity::cosine(embedding, $query) >= $min_similarity + ORDER BY similarity DESC + LIMIT $match_count + )} + ELSE { [] }; + + + let $all_results = array::union( + array::union($source_embedding_search, $source_insight_search), + $note_content_search + ); + + + RETURN (select id, parent_id, title, math::max(similarity) as similarity, + array::flatten(content) as matches + from $all_results where id is not None + group by id, parent_id, title ORDER BY similarity DESC LIMIT $match_count); + +}; \ No newline at end of file diff --git a/open_notebook/database/repository.py b/open_notebook/database/repository.py new file mode 100644 index 0000000000000000000000000000000000000000..3fb9e04e80d0978e15f3e4572945f9d7266d2e99 --- /dev/null +++ b/open_notebook/database/repository.py @@ -0,0 +1,194 @@ +import os +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, TypeVar, Union + +from loguru import logger +from surrealdb import AsyncSurreal, RecordID # type: ignore + +T = TypeVar("T", Dict[str, Any], List[Dict[str, Any]]) + + +def get_database_url(): + """Get database URL with backward compatibility""" + surreal_url = os.getenv("SURREAL_URL") + if surreal_url: + return surreal_url + + # Fallback to old format - WebSocket URL format + address = os.getenv("SURREAL_ADDRESS", "localhost") + port = os.getenv("SURREAL_PORT", "8000") + return f"ws://{address}/rpc:{port}" + + +def get_database_password(): + """Get password with backward compatibility""" + return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS") + + +def parse_record_ids(obj: Any) -> Any: + """Recursively parse and convert RecordIDs into strings.""" + if isinstance(obj, dict): + return {k: parse_record_ids(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [parse_record_ids(item) for item in obj] + elif isinstance(obj, RecordID): + return str(obj) + return obj + + +def ensure_record_id(value: Union[str, RecordID]) -> RecordID: + """Ensure a value is a RecordID.""" + if isinstance(value, RecordID): + return value + return RecordID.parse(value) + + +@asynccontextmanager +async def db_connection(): + db = AsyncSurreal(get_database_url()) + await db.signin( + { + "username": os.environ.get("SURREAL_USER"), + "password": get_database_password(), + } + ) + await db.use( + os.environ.get("SURREAL_NAMESPACE"), os.environ.get("SURREAL_DATABASE") + ) + try: + yield db + finally: + await db.close() + + +async def repo_query( + query_str: str, vars: Optional[Dict[str, Any]] = None +) -> List[Dict[str, Any]]: + """Execute a SurrealQL query and return the results""" + + async with db_connection() as connection: + try: + result = parse_record_ids(await connection.query(query_str, vars)) + if isinstance(result, str): + raise RuntimeError(result) + return result + except RuntimeError as e: + # RuntimeError is raised for retriable transaction conflicts - log at debug to avoid noise + logger.debug(str(e)) + raise + except Exception as e: + logger.exception(e) + raise + + +async def repo_create(table: str, data: Dict[str, Any]) -> Dict[str, Any]: + """Create a new record in the specified table""" + # Remove 'id' attribute if it exists in data + data.pop("id", None) + data["created"] = datetime.now(timezone.utc) + data["updated"] = datetime.now(timezone.utc) + try: + async with db_connection() as connection: + result = parse_record_ids(await connection.insert(table, data)) + # SurrealDB may return a string error message instead of the expected record + if isinstance(result, str): + raise RuntimeError(result) + return result + except RuntimeError as e: + logger.error(str(e)) + raise + except Exception as e: + logger.exception(e) + raise RuntimeError("Failed to create record") + + +async def repo_relate( + source: str, relationship: str, target: str, data: Optional[Dict[str, Any]] = None +) -> List[Dict[str, Any]]: + """Create a relationship between two records with optional data""" + if data is None: + data = {} + query = f"RELATE {source}->{relationship}->{target} CONTENT $data;" + # logger.debug(f"Relate query: {query}") + + return await repo_query( + query, + { + "data": data, + }, + ) + + +async def repo_upsert( + table: str, id: Optional[str], data: Dict[str, Any], add_timestamp: bool = False +) -> List[Dict[str, Any]]: + """Create or update a record in the specified table""" + data.pop("id", None) + if add_timestamp: + data["updated"] = datetime.now(timezone.utc) + query = f"UPSERT {id if id else table} MERGE $data;" + return await repo_query(query, {"data": data}) + + +async def repo_update( + table: str, id: str, data: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Update an existing record by table and id""" + # If id already contains the table name, use it as is + try: + if isinstance(id, RecordID) or (":" in id and id.startswith(f"{table}:")): + record_id = id + else: + record_id = f"{table}:{id}" + data.pop("id", None) + if "created" in data and isinstance(data["created"], str): + data["created"] = datetime.fromisoformat(data["created"]) + data["updated"] = datetime.now(timezone.utc) + query = f"UPDATE {record_id} MERGE $data;" + # logger.debug(f"Update query: {query}") + result = await repo_query(query, {"data": data}) + # if isinstance(result, list): + # return [_return_data(item) for item in result] + return parse_record_ids(result) + except Exception as e: + raise RuntimeError(f"Failed to update record: {str(e)}") + + +async def repo_delete(record_id: Union[str, RecordID]): + """Delete a record by record id""" + + try: + async with db_connection() as connection: + return await connection.delete(ensure_record_id(record_id)) + except Exception as e: + logger.exception(e) + raise RuntimeError(f"Failed to delete record: {str(e)}") + + +async def repo_insert( + table: str, data: List[Dict[str, Any]], ignore_duplicates: bool = False +) -> List[Dict[str, Any]]: + """Create a new record in the specified table""" + try: + async with db_connection() as connection: + result = parse_record_ids(await connection.insert(table, data)) + # SurrealDB may return a string error message instead of the expected records + if isinstance(result, str): + raise RuntimeError(result) + return result + except RuntimeError as e: + if ignore_duplicates and "already contains" in str(e): + return [] + # Log transaction conflicts at debug level (they are expected during concurrent operations) + error_str = str(e).lower() + if "transaction" in error_str or "conflict" in error_str: + logger.debug(str(e)) + else: + logger.error(str(e)) + raise + except Exception as e: + if ignore_duplicates and "already contains" in str(e): + return [] + logger.exception(e) + raise RuntimeError("Failed to create record") diff --git a/open_notebook/domain/CLAUDE.md b/open_notebook/domain/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..79678f47691e76e4b5891acc8933f6b10bc8004f --- /dev/null +++ b/open_notebook/domain/CLAUDE.md @@ -0,0 +1,122 @@ +# Domain Module + +Core data models for notebooks, sources, notes, and settings with async SurrealDB persistence, auto-embedding, and relationship management. + +## Purpose + +Two base classes support different persistence patterns: **ObjectModel** (mutable records with auto-increment IDs) and **RecordModel** (singleton configuration with fixed IDs). + +## Key Components + +### base.py +- **ObjectModel**: Base for notebooks, sources, notes + - `save()`: Create/update with auto-embedding for searchable content + - `delete()`: Remove by ID + - `relate(relationship, target_id)`: Create graph relationships (reference, artifact, refers_to) + - `get(id)`: Polymorphic fetch; resolves subclass from ID prefix + - `get_all(order_by)`: Fetch all records from table + - Integrates with ModelManager for automatic embedding + +- **RecordModel**: Singleton configuration (ContentSettings, DefaultPrompts) + - Fixed record_id per subclass + - `update()`: Upsert to database + - Lazy DB loading via `_load_from_db()` + +### notebook.py +- **Notebook**: Research project container + - `get_sources()`, `get_notes()`, `get_chat_sessions()`: Navigate relationships + - `get_delete_preview()`: Returns counts of notes, exclusive sources, and shared sources that would be affected by deletion + - `delete(delete_exclusive_sources)`: Cascade deletion - always deletes notes, optionally deletes exclusive sources, always unlinks all sources + +- **Source**: Content item (file/URL) + - `vectorize()`: Submit async embedding job (returns command_id, fire-and-forget) + - `get_status()`, `get_processing_progress()`: Track job via surreal_commands + - `get_context()`: Returns summary for LLM context + - `add_insight()`: Submit async insight creation via `create_insight_command` (fire-and-forget, returns command_id) + +- **Note**: Standalone or linked notes + - `save()`: Submits `embed_note` command after save (fire-and-forget) + - `add_to_notebook()`: Link to notebook + +- **SourceInsight, SourceEmbedding**: Derived content models +- **ChatSession**: Conversation container with optional model_override +- **Asset**: File/URL reference helper + +- **Search functions**: + - `text_search()`: Full-text keyword search + - `vector_search()`: Semantic search via embeddings (default minimum_score=0.2) + +### content_settings.py +- **ContentSettings**: Singleton for processing engines, embedding strategy, file deletion, YouTube languages + +### transformation.py +- **Transformation**: Reusable prompts for content transformation +- **DefaultPrompts**: Singleton with transformation instructions + +### credential.py +- **Credential**: Individual credential records for API keys and provider configuration + - **One record per credential**: Each credential (e.g., "My OpenAI Key", "Work Anthropic") is a separate `Credential` record in SurrealDB + - **Fields**: name, provider, modalities (list), api_key (SecretStr), base_url, endpoint, api_version, endpoint_llm/embedding/stt/tts, project, location, credentials_path + - **SecretStr protection**: API key field uses Pydantic's `SecretStr` (values masked in logs/repr) + - **Encryption integration**: Uses `encrypt_value()`/`decrypt_value()` from `open_notebook.utils.encryption` + - Keys encrypted with Fernet before database storage + - Requires `OPEN_NOTEBOOK_ENCRYPTION_KEY` environment variable (warns if not set) + - **Key methods**: + - `to_esperanto_config()`: Builds config dict for Esperanto's AIFactory methods + - `get_by_provider(provider)`: Class method to fetch all credentials for a provider + - `get_linked_models()`: Returns all Model records linked to this credential + - **Custom serialization**: `_prepare_save_data()` extracts SecretStr values and encrypts before storage + - **Decryption on read**: `get()` and `get_all()` overridden to decrypt api_key after fetch + +- **Note**: `provider_config.py` still exists for legacy migration support (migrating old ProviderConfig records to Credential) + +## Important Patterns + +- **Async/await**: All DB operations async; always use await +- **Polymorphic get()**: `ObjectModel.get(id)` determines subclass from ID prefix (table:id format) +- **Fire-and-forget embedding**: Models submit embed_* commands after save via `submit_command()` (non-blocking) +- **Nullable fields**: Declare via `nullable_fields` ClassVar to allow None in database +- **Timestamps**: `created` and `updated` auto-managed as ISO strings +- **Fire-and-forget jobs**: `source.vectorize()` returns command_id without waiting + +## Key Dependencies + +- `surrealdb`: RecordID type for relationships +- `pydantic`: Validation and field_validator decorators +- `open_notebook.database.repository`: CRUD and relationship functions +- `open_notebook.ai.models`: ModelManager for embeddings +- `surreal_commands`: Async job submission (vectorization, insights) +- `loguru`: Logging + +## Quirks & Gotchas + +- **Polymorphic resolution**: `ObjectModel.get()` fails if subclass not imported (search subclasses list) +- **RecordModel singleton**: __new__ returns existing instance; call `clear_instance()` in tests +- **Source.command field**: Stored as RecordID; auto-parsed from strings via field_validator +- **Text truncation**: `Note.get_context(short)` hardcodes 100-char limit +- **Auto-embedding behavior**: + - `Note.save()` → auto-submits `embed_note` command + - `Source.save()` → does NOT auto-submit (must call `vectorize()` explicitly) + - `Source.add_insight()` → submits `create_insight_command` which handles DB insert + `embed_insight` command (all fire-and-forget) +- **Relationship strings**: Must match SurrealDB schema (reference, artifact, refers_to) + +## How to Add New Model + +1. Inherit from ObjectModel with table_name ClassVar +2. Define Pydantic fields with validators +3. Override `save()` to submit embedding command if searchable (use `submit_command("embed_*", id)`) +4. Add custom methods for domain logic (get_X, add_to_Y) +5. Implement `_prepare_save_data()` if custom serialization needed + +## Usage + +```python +notebook = Notebook(name="Research", description="My project") +await notebook.save() + +obj = await ObjectModel.get("notebook:123") # Polymorphic fetch + +# Search +await text_search("quantum", results=5) +await vector_search("quantum computing", results=10, minimum_score=0.3) +``` diff --git a/open_notebook/domain/__init__.py b/open_notebook/domain/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9de250587492a161e561210f0d851afce0576a6 --- /dev/null +++ b/open_notebook/domain/__init__.py @@ -0,0 +1,7 @@ +""" +Domain models for Open Notebook. + +This module exports the core domain models used throughout the application. +""" + +__all__: list[str] = [] diff --git a/open_notebook/domain/base.py b/open_notebook/domain/base.py new file mode 100644 index 0000000000000000000000000000000000000000..45459d45af4ff7ca9e758c7d0d6e280f8bf16fa0 --- /dev/null +++ b/open_notebook/domain/base.py @@ -0,0 +1,361 @@ +from datetime import datetime +from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar, Union, cast + +from loguru import logger +from pydantic import ( + BaseModel, + ConfigDict, + ValidationError, + field_validator, + model_validator, +) + +from open_notebook.database.repository import ( + ensure_record_id, + repo_create, + repo_delete, + repo_query, + repo_relate, + repo_update, + repo_upsert, +) +from open_notebook.exceptions import ( + DatabaseOperationError, + InvalidInputError, + NotFoundError, +) + +T = TypeVar("T", bound="ObjectModel") + + +class ObjectModel(BaseModel): + id: Optional[str] = None + table_name: ClassVar[str] = "" + nullable_fields: ClassVar[set[str]] = set() # Fields that can be saved as None + created: Optional[datetime] = None + updated: Optional[datetime] = None + + @classmethod + async def get_all(cls: Type[T], order_by=None) -> List[T]: + try: + # If called from a specific subclass, use its table_name + if cls.table_name: + target_class = cls + table_name = cls.table_name + else: + # This path is taken if called directly from ObjectModel + raise InvalidInputError( + "get_all() must be called from a specific model class" + ) + if order_by: + # Validate order_by to prevent SurrealQL injection + # Supports: "field", "field direction", "field1 direction, field2 direction" + import re + + allowed_field_pattern = re.compile(r"^[a-z_][a-z0-9_]*$") + allowed_directions = {"asc", "desc"} + + clauses = [c.strip() for c in order_by.split(",")] + validated_clauses = [] + for clause in clauses: + parts = clause.strip().split() + if len(parts) == 1: + if not allowed_field_pattern.match(parts[0].lower()): + raise InvalidInputError( + f"Invalid order_by field: '{parts[0]}'" + ) + validated_clauses.append(parts[0].lower()) + elif len(parts) == 2: + if not allowed_field_pattern.match( + parts[0].lower() + ) or parts[1].lower() not in allowed_directions: + raise InvalidInputError( + f"Invalid order_by clause: '{clause.strip()}'" + ) + validated_clauses.append( + f"{parts[0].lower()} {parts[1].lower()}" + ) + else: + raise InvalidInputError( + f"Invalid order_by clause: '{clause.strip()}'" + ) + + validated_order_by = ", ".join(validated_clauses) + query = f"SELECT * FROM {table_name} ORDER BY {validated_order_by}" + else: + query = f"SELECT * FROM {table_name}" + + result = await repo_query(query) + objects = [] + for obj in result: + try: + objects.append(target_class(**obj)) + except Exception as e: + logger.critical(f"Error creating object: {str(e)}") + + return objects + except Exception as e: + logger.error(f"Error fetching all {cls.table_name}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + @classmethod + async def get(cls: Type[T], id: str) -> T: + if not id: + raise InvalidInputError("ID cannot be empty") + try: + # Get the table name from the ID (everything before the first colon) + table_name = id.split(":")[0] if ":" in id else id + + # If we're calling from a specific subclass and IDs match, use that class + if cls.table_name and cls.table_name == table_name: + target_class: Type[T] = cls + else: + # Otherwise, find the appropriate subclass based on table_name + found_class = cls._get_class_by_table_name(table_name) + if not found_class: + raise InvalidInputError(f"No class found for table {table_name}") + target_class = cast(Type[T], found_class) + + result = await repo_query("SELECT * FROM $id", {"id": ensure_record_id(id)}) + if result: + return target_class(**result[0]) + else: + raise NotFoundError(f"{table_name} with id {id} not found") + except Exception as e: + logger.error(f"Error fetching object with id {id}: {str(e)}") + logger.exception(e) + raise NotFoundError(f"Object with id {id} not found - {str(e)}") + + @classmethod + def _get_class_by_table_name(cls, table_name: str) -> Optional[Type["ObjectModel"]]: + """Find the appropriate subclass based on table_name.""" + + def get_all_subclasses(c: Type["ObjectModel"]) -> List[Type["ObjectModel"]]: + all_subclasses: List[Type["ObjectModel"]] = [] + for subclass in c.__subclasses__(): + all_subclasses.append(subclass) + all_subclasses.extend(get_all_subclasses(subclass)) + return all_subclasses + + for subclass in get_all_subclasses(ObjectModel): + if hasattr(subclass, "table_name") and subclass.table_name == table_name: + return subclass + return None + + async def save(self) -> None: + """ + Save the model to the database. + + Note: Embedding is no longer generated inline. Subclasses that need + embedding should override save() to submit the appropriate embed_* + command after calling super().save(). + """ + try: + self.model_validate(self.model_dump(), strict=True) + data = self._prepare_save_data() + data["updated"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + repo_result: Union[List[Dict[str, Any]], Dict[str, Any]] + if self.id is None: + data["created"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + repo_result = await repo_create(self.__class__.table_name, data) + else: + data["created"] = ( + self.created.strftime("%Y-%m-%d %H:%M:%S") + if isinstance(self.created, datetime) + else self.created + ) + logger.debug(f"Updating record with id {self.id}") + repo_result = await repo_update( + self.__class__.table_name, self.id, data + ) + # Update the current instance with the result + # repo_result is a list of dictionaries + result_list: List[Dict[str, Any]] = ( + repo_result if isinstance(repo_result, list) else [repo_result] + ) + for key, value in result_list[0].items(): + if hasattr(self, key): + if isinstance(getattr(self, key), BaseModel): + setattr(self, key, type(getattr(self, key))(**value)) + else: + setattr(self, key, value) + + except ValidationError as e: + logger.error(f"Validation failed: {e}") + raise + except RuntimeError: + # Transaction conflicts should propagate for retry + raise + except Exception as e: + logger.error(f"Error saving record: {e}") + raise DatabaseOperationError(e) + + def _prepare_save_data(self) -> Dict[str, Any]: + data = self.model_dump() + return { + key: value + for key, value in data.items() + if value is not None or key in self.__class__.nullable_fields + } + + async def delete(self) -> bool: + if self.id is None: + raise InvalidInputError("Cannot delete object without an ID") + try: + logger.debug(f"Deleting record with id {self.id}") + return await repo_delete(self.id) + except Exception as e: + logger.error( + f"Error deleting {self.__class__.table_name} with id {self.id}: {str(e)}" + ) + raise DatabaseOperationError( + f"Failed to delete {self.__class__.table_name}" + ) + + async def relate( + self, relationship: str, target_id: str, data: Optional[Dict] = {} + ) -> Any: + if not relationship or not target_id or not self.id: + raise InvalidInputError("Relationship and target ID must be provided") + try: + return await repo_relate( + source=self.id, relationship=relationship, target=target_id, data=data + ) + except Exception as e: + logger.error(f"Error creating relationship: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + @field_validator("created", "updated", mode="before") + @classmethod + def parse_datetime(cls, value): + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + return value + + +class RecordModel(BaseModel): + model_config = ConfigDict( + validate_assignment=True, + arbitrary_types_allowed=True, + extra="allow", + from_attributes=True, + defer_build=True, + ) + + record_id: ClassVar[str] + auto_save: ClassVar[bool] = ( + False # Default to False, can be overridden in subclasses + ) + _instances: ClassVar[Dict[str, "RecordModel"]] = {} # Store instances by record_id + + def __new__(cls, **kwargs): + # If an instance already exists for this record_id, return it + if cls.record_id in cls._instances: + instance = cls._instances[cls.record_id] + # Update instance with any new kwargs if provided + if kwargs: + for key, value in kwargs.items(): + setattr(instance, key, value) + return instance + + # If no instance exists, create a new one + instance = super().__new__(cls) + cls._instances[cls.record_id] = instance + return instance + + def __init__(self, **kwargs): + # Only initialize if this is a new instance + if not hasattr(self, "_initialized"): + object.__setattr__(self, "__dict__", {}) + + # For RecordModel, we need to handle async initialization differently + # Initialize with provided kwargs only for now + super().__init__(**kwargs) + + # Mark as initialized but not loaded from DB yet + object.__setattr__(self, "_initialized", True) + object.__setattr__(self, "_db_loaded", False) + + async def _load_from_db(self): + """Load data from database if not already loaded""" + if not getattr(self, "_db_loaded", False): + result = await repo_query( + "SELECT * FROM ONLY $record_id", + {"record_id": ensure_record_id(self.record_id)}, + ) + + # Handle case where record doesn't exist yet + if result: + if isinstance(result, list) and len(result) > 0: + # Standard list response + row = result[0] + if isinstance(row, dict): + for key, value in row.items(): + if hasattr(self, key): + object.__setattr__(self, key, value) + elif isinstance(result, dict): + # Direct dict response + for key, value in result.items(): + if hasattr(self, key): + object.__setattr__(self, key, value) + + object.__setattr__(self, "_db_loaded", True) + + @classmethod + async def get_instance(cls) -> "RecordModel": + """Get or create the singleton instance and load from DB""" + instance = cls() + await instance._load_from_db() + return instance + + @model_validator(mode="after") + def auto_save_validator(self): + if self.__class__.auto_save: + # Auto-save can't work with async - log warning + logger.warning( + f"Auto-save is enabled for {self.__class__.__name__} but update() is now async. Call await instance.update() manually." + ) + return self + + async def update(self): + # Get all non-ClassVar fields and their values + data = { + field_name: getattr(self, field_name) + for field_name, field_info in self.model_fields.items() + if not str(field_info.annotation).startswith("typing.ClassVar") + } + + await repo_upsert( + self.__class__.table_name + if hasattr(self.__class__, "table_name") + else "record", + self.record_id, + data, + ) + + result = await repo_query( + "SELECT * FROM $record_id", {"record_id": ensure_record_id(self.record_id)} + ) + if result: + for key, value in result[0].items(): + if hasattr(self, key): + object.__setattr__( + self, key, value + ) # Use object.__setattr__ to avoid triggering validation again + + return self + + @classmethod + def clear_instance(cls): + """Clear the singleton instance (useful for testing)""" + if cls.record_id in cls._instances: + del cls._instances[cls.record_id] + + async def patch(self, model_dict: dict): + """Update model attributes from dictionary and save""" + for key, value in model_dict.items(): + setattr(self, key, value) + await self.update() diff --git a/open_notebook/domain/content_settings.py b/open_notebook/domain/content_settings.py new file mode 100644 index 0000000000000000000000000000000000000000..bb79f89861b26d725ece0825a99d6f62b33269fb --- /dev/null +++ b/open_notebook/domain/content_settings.py @@ -0,0 +1,25 @@ +from typing import ClassVar, List, Literal, Optional + +from pydantic import Field + +from open_notebook.domain.base import RecordModel + + +class ContentSettings(RecordModel): + record_id: ClassVar[str] = "open_notebook:content_settings" + default_content_processing_engine_doc: Optional[ + Literal["auto", "docling", "simple"] + ] = Field("auto", description="Default Content Processing Engine for Documents") + default_content_processing_engine_url: Optional[ + Literal["auto", "firecrawl", "jina", "simple"] + ] = Field("auto", description="Default Content Processing Engine for URLs") + default_embedding_option: Optional[Literal["ask", "always", "never"]] = Field( + "ask", description="Default Embedding Option for Vector Search" + ) + auto_delete_files: Optional[Literal["yes", "no"]] = Field( + "yes", description="Auto Delete Uploaded Files" + ) + youtube_preferred_languages: Optional[List[str]] = Field( + ["en", "pt", "es", "de", "nl", "en-GB", "fr", "de", "hi", "ja"], + description="Preferred languages for YouTube transcripts", + ) diff --git a/open_notebook/domain/credential.py b/open_notebook/domain/credential.py new file mode 100644 index 0000000000000000000000000000000000000000..16e5ae7ca75822564fab661039b9287156adfce4 --- /dev/null +++ b/open_notebook/domain/credential.py @@ -0,0 +1,234 @@ +""" +Credential domain model for storing individual provider credentials. + +Each credential is a standalone record in the 'credential' table, replacing +the old ProviderConfig singleton. Credentials store API keys (encrypted at +rest) and provider-specific configuration fields. + +Usage: + cred = Credential( + name="Production", + provider="openai", + modalities=["language", "embedding"], + api_key=SecretStr("sk-..."), + ) + await cred.save() +""" + +from datetime import datetime +from typing import Any, ClassVar, Dict, List, Optional + +from loguru import logger +from pydantic import SecretStr + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.base import ObjectModel +from open_notebook.utils.encryption import decrypt_value, encrypt_value + + +class Credential(ObjectModel): + """ + Individual credential record for an AI provider. + + Each record stores authentication and configuration for a single provider + account. Models link to credentials via the credential field. + """ + + table_name: ClassVar[str] = "credential" + nullable_fields: ClassVar[set[str]] = { + "api_key", + "base_url", + "endpoint", + "api_version", + "endpoint_llm", + "endpoint_embedding", + "endpoint_stt", + "endpoint_tts", + "project", + "location", + "credentials_path", + } + + name: str + provider: str + modalities: List[str] = [] + api_key: Optional[SecretStr] = None + decryption_error: Optional[str] = None + base_url: Optional[str] = None + endpoint: Optional[str] = None + api_version: Optional[str] = None + endpoint_llm: Optional[str] = None + endpoint_embedding: Optional[str] = None + endpoint_stt: Optional[str] = None + endpoint_tts: Optional[str] = None + project: Optional[str] = None + location: Optional[str] = None + credentials_path: Optional[str] = None + + def to_esperanto_config(self) -> Dict[str, Any]: + """ + Build config dict for AIFactory.create_*() calls. + + Returns a dict that can be passed as the 'config' parameter to + Esperanto's AIFactory methods, overriding env var lookup. + """ + config: Dict[str, Any] = {} + if self.api_key: + config["api_key"] = self.api_key.get_secret_value() + if self.base_url: + config["base_url"] = self.base_url + # For Azure, base_url from the UI form maps to endpoint + if self.provider and self.provider.lower() == "azure" and not self.endpoint: + config["endpoint"] = self.base_url + if self.endpoint: + config["endpoint"] = self.endpoint + if self.api_version: + config["api_version"] = self.api_version + if self.endpoint_llm: + config["endpoint_llm"] = self.endpoint_llm + if self.endpoint_embedding: + config["endpoint_embedding"] = self.endpoint_embedding + if self.endpoint_stt: + config["endpoint_stt"] = self.endpoint_stt + if self.endpoint_tts: + config["endpoint_tts"] = self.endpoint_tts + if self.project: + config["project"] = self.project + if self.location: + config["location"] = self.location + if self.credentials_path: + config["credentials_path"] = self.credentials_path + return config + + @classmethod + async def get_by_provider(cls, provider: str) -> List["Credential"]: + """Get all credentials for a provider.""" + results = await repo_query( + "SELECT * FROM credential WHERE string::lowercase(provider) = string::lowercase($provider) ORDER BY created ASC", + {"provider": provider}, + ) + credentials = [] + for row in results: + try: + cred = cls._from_db_row(row) + credentials.append(cred) + except Exception as e: + logger.warning(f"Skipping invalid credential: {e}") + return credentials + + @classmethod + async def get(cls, id: str) -> "Credential": + """Override get() to handle api_key decryption.""" + instance = await super().get(id) + # Pydantic auto-wraps the raw DB string in SecretStr, so we need + # to extract, decrypt, and re-wrap regardless of type. + if instance.api_key: + raw = ( + instance.api_key.get_secret_value() + if isinstance(instance.api_key, SecretStr) + else instance.api_key + ) + decrypted = decrypt_value(raw) + object.__setattr__(instance, "api_key", SecretStr(decrypted)) + return instance + + @classmethod + async def get_all(cls, order_by=None) -> List["Credential"]: + """Override get_all() to handle api_key decryption with per-row error handling.""" + order_clause = f" ORDER BY {order_by}" if order_by else "" + results = await repo_query( + f"SELECT * FROM {cls.table_name}{order_clause}", + {}, + ) + credentials = [] + for row in results: + try: + cred = cls._from_db_row(row) + credentials.append(cred) + except Exception as e: + logger.warning( + f"Failed to decrypt credential {row.get('id', 'unknown')}: {e}" + ) + # Create a minimal credential with error info from raw DB fields + try: + error_cred = cls( + name=row.get("name", "Unknown"), + provider=row.get("provider", "unknown"), + modalities=row.get("modalities", []), + decryption_error="Failed to decrypt API key. The encryption key may have changed.", + ) + # Preserve the DB id, created, updated from the raw row + if row.get("id"): + object.__setattr__(error_cred, "id", str(row["id"])) + if row.get("created"): + object.__setattr__(error_cred, "created", row["created"]) + if row.get("updated"): + object.__setattr__(error_cred, "updated", row["updated"]) + # Mark that it had an api_key (even though we can't decrypt it) + if row.get("api_key"): + object.__setattr__( + error_cred, "api_key", SecretStr("UNDECRYPTABLE") + ) + credentials.append(error_cred) + except Exception as inner_e: + logger.error( + f"Failed to create error credential for {row.get('id', 'unknown')}: {inner_e}" + ) + return credentials + + async def get_linked_models(self) -> list: + """Get all models linked to this credential.""" + if not self.id: + return [] + from open_notebook.ai.models import Model + + results = await repo_query( + "SELECT * FROM model WHERE credential = $cred_id", + {"cred_id": ensure_record_id(self.id)}, + ) + return [Model(**row) for row in results] + + def _prepare_save_data(self) -> Dict[str, Any]: + """Override to encrypt api_key before storage.""" + data = {} + for key, value in self.model_dump().items(): + if key == "decryption_error": + continue + if key == "api_key": + # Handle SecretStr: extract, encrypt, store + if self.api_key: + secret_value = self.api_key.get_secret_value() + data["api_key"] = encrypt_value(secret_value) + else: + data["api_key"] = None + elif value is not None or key in self.__class__.nullable_fields: + data[key] = value + + return data + + async def save(self) -> None: + """Save credential, handling api_key re-hydration after DB round-trip.""" + # Remember the original SecretStr before save + original_api_key = self.api_key + + await super().save() + + # After save, the api_key field may be set to the encrypted string + # from the DB result. Restore the original SecretStr. + if original_api_key: + object.__setattr__(self, "api_key", original_api_key) + elif self.api_key and isinstance(self.api_key, str): + # Decrypt if DB returned an encrypted string + decrypted = decrypt_value(self.api_key) + object.__setattr__(self, "api_key", SecretStr(decrypted)) + + @classmethod + def _from_db_row(cls, row: dict) -> "Credential": + """Create a Credential from a database row, decrypting api_key.""" + api_key_val = row.get("api_key") + if api_key_val and isinstance(api_key_val, str): + decrypted = decrypt_value(api_key_val) + row["api_key"] = SecretStr(decrypted) + elif api_key_val is None: + row["api_key"] = None + return cls(**row) diff --git a/open_notebook/domain/notebook.py b/open_notebook/domain/notebook.py new file mode 100644 index 0000000000000000000000000000000000000000..3c4ea22182aa50361036d3041502e4746dec716e --- /dev/null +++ b/open_notebook/domain/notebook.py @@ -0,0 +1,680 @@ +import asyncio +import os +from pathlib import Path +from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Union + +from loguru import logger +from pydantic import BaseModel, ConfigDict, Field, field_validator +from surreal_commands import submit_command +from surrealdb import RecordID + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.base import ObjectModel +from open_notebook.exceptions import DatabaseOperationError, InvalidInputError + + +class Notebook(ObjectModel): + table_name: ClassVar[str] = "notebook" + name: str + description: str + archived: Optional[bool] = False + + @field_validator("name") + @classmethod + def name_must_not_be_empty(cls, v): + if not v.strip(): + raise InvalidInputError("Notebook name cannot be empty") + return v + + async def get_sources(self) -> List["Source"]: + try: + srcs = await repo_query( + """ + select * omit source.full_text from ( + select in as source from reference where out=$id + fetch source + ) order by source.updated desc + """, + {"id": ensure_record_id(self.id)}, + ) + return [Source(**src["source"]) for src in srcs] if srcs else [] + except Exception as e: + logger.error(f"Error fetching sources for notebook {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + async def get_notes(self) -> List["Note"]: + try: + srcs = await repo_query( + """ + select * omit note.content, note.embedding from ( + select in as note from artifact where out=$id + fetch note + ) order by note.updated desc + """, + {"id": ensure_record_id(self.id)}, + ) + return [Note(**src["note"]) for src in srcs] if srcs else [] + except Exception as e: + logger.error(f"Error fetching notes for notebook {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + async def get_chat_sessions(self) -> List["ChatSession"]: + try: + srcs = await repo_query( + """ + select * from ( + select + <- chat_session as chat_session + from refers_to + where out=$id + fetch chat_session + ) + order by chat_session.updated desc + """, + {"id": ensure_record_id(self.id)}, + ) + return ( + [ChatSession(**src["chat_session"][0]) for src in srcs] if srcs else [] + ) + except Exception as e: + logger.error( + f"Error fetching chat sessions for notebook {self.id}: {str(e)}" + ) + logger.exception(e) + raise DatabaseOperationError(e) + + async def get_delete_preview(self) -> Dict[str, Any]: + """ + Get counts of items that would be affected by deleting this notebook. + + Returns a dict with: + - note_count: Number of notes that will be deleted + - exclusive_source_count: Sources only in this notebook (can be deleted) + - shared_source_count: Sources in other notebooks (will be unlinked only) + """ + try: + notebook_id = ensure_record_id(self.id) + + # Count notes + note_result = await repo_query( + "SELECT count() as count FROM artifact WHERE out = $notebook_id GROUP ALL", + {"notebook_id": notebook_id}, + ) + note_count = note_result[0]["count"] if note_result else 0 + + # Get sources with count of references to OTHER notebooks + # If assigned_others = 0, source is exclusive to this notebook + # If assigned_others > 0, source is shared with other notebooks + source_counts = await repo_query( + """ + SELECT + id, + count(->reference[WHERE out != $notebook_id].out) as assigned_others + FROM (SELECT VALUE <-reference.in AS sources FROM $notebook_id)[0] + """, + {"notebook_id": notebook_id}, + ) + + exclusive_count = 0 + shared_count = 0 + for src in source_counts: + if src.get("assigned_others", 0) == 0: + exclusive_count += 1 + else: + shared_count += 1 + + return { + "note_count": note_count, + "exclusive_source_count": exclusive_count, + "shared_source_count": shared_count, + } + except Exception as e: + logger.error(f"Error getting delete preview for notebook {self.id}: {e}") + logger.exception(e) + raise DatabaseOperationError(e) + + async def delete(self, delete_exclusive_sources: bool = False) -> Dict[str, int]: + """ + Delete notebook with cascade deletion of notes and optional source deletion. + + Args: + delete_exclusive_sources: If True, also delete sources that belong + only to this notebook. Default is False. + + Returns: + Dict with counts: deleted_notes, deleted_sources, unlinked_sources + """ + if self.id is None: + raise InvalidInputError("Cannot delete notebook without an ID") + + try: + notebook_id = ensure_record_id(self.id) + deleted_notes = 0 + deleted_sources = 0 + unlinked_sources = 0 + + # 1. Get and delete all notes linked to this notebook + notes = await self.get_notes() + for note in notes: + await note.delete() + deleted_notes += 1 + logger.info(f"Deleted {deleted_notes} notes for notebook {self.id}") + + # Delete artifact relationships + await repo_query( + "DELETE artifact WHERE out = $notebook_id", + {"notebook_id": notebook_id}, + ) + + # 2. Handle sources + if delete_exclusive_sources: + # Find sources with count of references to OTHER notebooks + # If assigned_others = 0, source is exclusive to this notebook + source_counts = await repo_query( + """ + SELECT + id, + count(->reference[WHERE out != $notebook_id].out) as assigned_others + FROM (SELECT VALUE <-reference.in AS sources FROM $notebook_id)[0] + """, + {"notebook_id": notebook_id}, + ) + + for src in source_counts: + source_id = src.get("id") + if source_id and src.get("assigned_others", 0) == 0: + # Exclusive source - delete it + try: + source = await Source.get(str(source_id)) + await source.delete() + deleted_sources += 1 + except Exception as e: + logger.warning( + f"Failed to delete exclusive source {source_id}: {e}" + ) + else: + unlinked_sources += 1 + else: + # Just count sources that will be unlinked + source_result = await repo_query( + "SELECT count() as count FROM reference WHERE out = $notebook_id GROUP ALL", + {"notebook_id": notebook_id}, + ) + unlinked_sources = source_result[0]["count"] if source_result else 0 + + # Delete reference relationships (unlink all sources) + await repo_query( + "DELETE reference WHERE out = $notebook_id", + {"notebook_id": notebook_id}, + ) + logger.info( + f"Unlinked {unlinked_sources} sources, deleted {deleted_sources} " + f"exclusive sources for notebook {self.id}" + ) + + # 3. Delete the notebook record itself + await super().delete() + logger.info(f"Deleted notebook {self.id}") + + return { + "deleted_notes": deleted_notes, + "deleted_sources": deleted_sources, + "unlinked_sources": unlinked_sources, + } + + except Exception as e: + logger.error(f"Error deleting notebook {self.id}: {e}") + logger.exception(e) + raise DatabaseOperationError(f"Failed to delete notebook: {e}") + + +class Asset(BaseModel): + file_path: Optional[str] = None + url: Optional[str] = None + + +class SourceEmbedding(ObjectModel): + table_name: ClassVar[str] = "source_embedding" + content: str + + async def get_source(self) -> "Source": + try: + src = await repo_query( + """ + select source.* from $id fetch source + """, + {"id": ensure_record_id(self.id)}, + ) + return Source(**src[0]["source"]) + except Exception as e: + logger.error(f"Error fetching source for embedding {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + +class SourceInsight(ObjectModel): + table_name: ClassVar[str] = "source_insight" + insight_type: str + content: str + + async def get_source(self) -> "Source": + try: + src = await repo_query( + """ + select source.* from $id fetch source + """, + {"id": ensure_record_id(self.id)}, + ) + return Source(**src[0]["source"]) + except Exception as e: + logger.error(f"Error fetching source for insight {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + async def save_as_note(self, notebook_id: Optional[str] = None) -> Any: + source = await self.get_source() + note = Note( + title=f"{self.insight_type} from source {source.title}", + content=self.content, + ) + await note.save() + if notebook_id: + await note.add_to_notebook(notebook_id) + return note + + +class Source(ObjectModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + table_name: ClassVar[str] = "source" + asset: Optional[Asset] = None + title: Optional[str] = None + topics: Optional[List[str]] = Field(default_factory=list) + full_text: Optional[str] = None + command: Optional[Union[str, RecordID]] = Field( + default=None, description="Link to surreal-commands processing job" + ) + + @field_validator("command", mode="before") + @classmethod + def parse_command(cls, value): + """Parse command field to ensure RecordID format""" + if isinstance(value, str) and value: + return ensure_record_id(value) + return value + + @field_validator("id", mode="before") + @classmethod + def parse_id(cls, value): + """Parse id field to handle both string and RecordID inputs""" + if value is None: + return None + if isinstance(value, RecordID): + return str(value) + return str(value) if value else None + + async def get_status(self) -> Optional[str]: + """Get the processing status of the associated command""" + if not self.command: + return None + + try: + from surreal_commands import get_command_status + + status = await get_command_status(str(self.command)) + return status.status if status else "unknown" + except Exception as e: + logger.warning(f"Failed to get command status for {self.command}: {e}") + return "unknown" + + async def get_processing_progress(self) -> Optional[Dict[str, Any]]: + """Get detailed processing information for the associated command""" + if not self.command: + return None + + try: + from surreal_commands import get_command_status + + status_result = await get_command_status(str(self.command)) + if not status_result: + return None + + # Extract execution metadata if available + result = getattr(status_result, "result", None) + execution_metadata = ( + result.get("execution_metadata", {}) if isinstance(result, dict) else {} + ) + + return { + "status": status_result.status, + "started_at": execution_metadata.get("started_at"), + "completed_at": execution_metadata.get("completed_at"), + "error": getattr(status_result, "error_message", None), + "result": result, + } + except Exception as e: + logger.warning(f"Failed to get command progress for {self.command}: {e}") + return None + + async def get_context( + self, context_size: Literal["short", "long"] = "short" + ) -> Dict[str, Any]: + insights_list = await self.get_insights() + insights = [insight.model_dump() for insight in insights_list] + if context_size == "long": + return dict( + id=self.id, + title=self.title, + insights=insights, + full_text=self.full_text, + ) + else: + return dict(id=self.id, title=self.title, insights=insights) + + async def get_embedded_chunks(self) -> int: + try: + result = await repo_query( + """ + select count() as chunks from source_embedding where source=$id GROUP ALL + """, + {"id": ensure_record_id(self.id)}, + ) + if len(result) == 0: + return 0 + return result[0]["chunks"] + except Exception as e: + logger.error(f"Error fetching chunks count for source {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(f"Failed to count chunks for source: {str(e)}") + + async def get_insights(self) -> List[SourceInsight]: + try: + result = await repo_query( + """ + SELECT * FROM source_insight WHERE source=$id + """, + {"id": ensure_record_id(self.id)}, + ) + return [SourceInsight(**insight) for insight in result] + except Exception as e: + logger.error(f"Error fetching insights for source {self.id}: {str(e)}") + logger.exception(e) + raise DatabaseOperationError("Failed to fetch insights for source") + + async def add_to_notebook(self, notebook_id: str) -> Any: + if not notebook_id: + raise InvalidInputError("Notebook ID must be provided") + return await self.relate("reference", notebook_id) + + async def vectorize(self) -> str: + """ + Submit vectorization as a background job using the embed_source command. + + This method leverages the job-based architecture to prevent HTTP connection + pool exhaustion when processing large documents. The embed_source command: + 1. Detects content type from file path + 2. Chunks text using content-type aware splitter + 3. Generates all embeddings in batches + 4. Bulk inserts source_embedding records + + Returns: + str: The command/job ID that can be used to track progress via the commands API + + Raises: + ValueError: If source has no text to vectorize + DatabaseOperationError: If job submission fails + """ + logger.info(f"Submitting embed_source job for source {self.id}") + + try: + if not self.full_text or not self.full_text.strip(): + raise ValueError(f"Source {self.id} has no text to vectorize") + + # Submit the embed_source command + command_id = submit_command( + "open_notebook", + "embed_source", + {"source_id": str(self.id)}, + ) + + command_id_str = str(command_id) + logger.info( + f"Embed source job submitted for source {self.id}: " + f"command_id={command_id_str}" + ) + + return command_id_str + + except ValueError: + raise + except Exception as e: + logger.error( + f"Failed to submit embed_source job for source {self.id}: {e}" + ) + logger.exception(e) + raise DatabaseOperationError(e) + + async def add_insight(self, insight_type: str, content: str) -> Optional[str]: + """ + Submit insight creation as an async command (fire-and-forget). + + Submits a create_insight command that handles database operations with + automatic retry logic for transaction conflicts. The command also submits + an embed_insight command for async embedding. + + This method returns immediately after submitting the command - it does NOT + wait for the insight to be created. Use this for batch operations where + throughput is more important than immediate confirmation. + + Args: + insight_type: Type/category of the insight + content: The insight content text + + Returns: + command_id for optional tracking, or None if submission failed + + Raises: + InvalidInputError: If insight_type or content is empty + """ + if not insight_type or not content: + raise InvalidInputError("Insight type and content must be provided") + + try: + # Submit create_insight command (fire-and-forget) + # Command handles retries internally for transaction conflicts + command_id = submit_command( + "open_notebook", + "create_insight", + { + "source_id": str(self.id), + "insight_type": insight_type, + "content": content, + }, + ) + logger.info( + f"Submitted create_insight command {command_id} for source {self.id} " + f"(type={insight_type})" + ) + return str(command_id) + + except Exception as e: + logger.error(f"Error submitting create_insight for source {self.id}: {e}") + return None + + def _prepare_save_data(self) -> dict: + """Override to ensure command field is always RecordID format for database""" + data = super()._prepare_save_data() + + # Ensure command field is RecordID format if not None + if data.get("command") is not None: + data["command"] = ensure_record_id(data["command"]) + + return data + + async def delete(self) -> bool: + """Delete source and clean up associated file, embeddings, and insights.""" + # Clean up uploaded file if it exists + if self.asset and self.asset.file_path: + file_path = Path(self.asset.file_path) + if file_path.exists(): + try: + os.unlink(file_path) + logger.info(f"Deleted file for source {self.id}: {file_path}") + except Exception as e: + logger.warning( + f"Failed to delete file {file_path} for source {self.id}: {e}. " + "Continuing with database deletion." + ) + else: + logger.debug( + f"File {file_path} not found for source {self.id}, skipping cleanup" + ) + + # Delete associated embeddings and insights to prevent orphaned records + try: + source_id = ensure_record_id(self.id) + await repo_query( + "DELETE source_embedding WHERE source = $source_id", + {"source_id": source_id}, + ) + await repo_query( + "DELETE source_insight WHERE source = $source_id", + {"source_id": source_id}, + ) + logger.debug(f"Deleted embeddings and insights for source {self.id}") + except Exception as e: + logger.warning( + f"Failed to delete embeddings/insights for source {self.id}: {e}. " + "Continuing with source deletion." + ) + + # Call parent delete to remove database record + return await super().delete() + + +class Note(ObjectModel): + table_name: ClassVar[str] = "note" + title: Optional[str] = None + note_type: Optional[Literal["human", "ai"]] = None + content: Optional[str] = None + + @field_validator("content") + @classmethod + def content_must_not_be_empty(cls, v): + if v is not None and not v.strip(): + raise InvalidInputError("Note content cannot be empty") + return v + + async def save(self) -> Optional[str]: + """ + Save the note and submit embedding command. + + Overrides ObjectModel.save() to submit an async embed_note command + after saving, instead of inline embedding. + + Returns: + Optional[str]: The command_id if embedding was submitted, None otherwise + """ + # Call parent save (without embedding) + await super().save() + + # Submit embedding command (fire-and-forget) if note has content + if self.id and self.content and self.content.strip(): + command_id = submit_command( + "open_notebook", + "embed_note", + {"note_id": str(self.id)}, + ) + logger.debug(f"Submitted embed_note command {command_id} for {self.id}") + return command_id + + return None + + async def add_to_notebook(self, notebook_id: str) -> Any: + if not notebook_id: + raise InvalidInputError("Notebook ID must be provided") + return await self.relate("artifact", notebook_id) + + def get_context( + self, context_size: Literal["short", "long"] = "short" + ) -> Dict[str, Any]: + if context_size == "long": + return dict(id=self.id, title=self.title, content=self.content) + else: + return dict( + id=self.id, + title=self.title, + content=self.content[:100] if self.content else None, + ) + + +class ChatSession(ObjectModel): + table_name: ClassVar[str] = "chat_session" + nullable_fields: ClassVar[set[str]] = {"model_override"} + title: Optional[str] = None + model_override: Optional[str] = None + + async def relate_to_notebook(self, notebook_id: str) -> Any: + if not notebook_id: + raise InvalidInputError("Notebook ID must be provided") + return await self.relate("refers_to", notebook_id) + + async def relate_to_source(self, source_id: str) -> Any: + if not source_id: + raise InvalidInputError("Source ID must be provided") + return await self.relate("refers_to", source_id) + + +async def text_search( + keyword: str, results: int, source: bool = True, note: bool = True +): + if not keyword: + raise InvalidInputError("Search keyword cannot be empty") + try: + search_results = await repo_query( + """ + select * + from fn::text_search($keyword, $results, $source, $note) + """, + {"keyword": keyword, "results": results, "source": source, "note": note}, + ) + return search_results + except Exception as e: + logger.error(f"Error performing text search: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) + + +async def vector_search( + keyword: str, + results: int, + source: bool = True, + note: bool = True, + minimum_score=0.2, +): + if not keyword: + raise InvalidInputError("Search keyword cannot be empty") + try: + from open_notebook.utils.embedding import generate_embedding + + # Use unified embedding function (handles chunking if query is very long) + embed = await generate_embedding(keyword) + search_results = await repo_query( + """ + SELECT * FROM fn::vector_search($embed, $results, $source, $note, $minimum_score); + """, + { + "embed": embed, + "results": results, + "source": source, + "note": note, + "minimum_score": minimum_score, + }, + ) + return search_results + except Exception as e: + logger.error(f"Error performing vector search: {str(e)}") + logger.exception(e) + raise DatabaseOperationError(e) diff --git a/open_notebook/domain/provider_config.py b/open_notebook/domain/provider_config.py new file mode 100644 index 0000000000000000000000000000000000000000..08a0d8a63cb11172604aa4e038487ce6f6cbc991 --- /dev/null +++ b/open_notebook/domain/provider_config.py @@ -0,0 +1,444 @@ +""" +Provider Configuration domain model for storing multiple credentials per provider. + +This module provides the ProviderConfig singleton model that stores multiple +API key configurations per provider. Each ProviderCredential contains a complete +set of configuration options for a provider (api_key, base_url, model, etc.). + +Encryption is enabled when OPEN_NOTEBOOK_ENCRYPTION_KEY environment variable +is set. If not set, keys are stored as plain text with a warning logged. +""" + +from datetime import datetime +from typing import ClassVar, Dict, List, Optional + +from pydantic import Field, SecretStr, field_validator + +from open_notebook.database.repository import ensure_record_id, repo_query, repo_upsert +from open_notebook.domain.base import RecordModel +from open_notebook.utils.encryption import decrypt_value, encrypt_value + + +class ProviderCredential: + """ + A single provider configuration item containing api_key and related settings. + + This class represents one complete configuration for an AI provider. + Multiple configurations can exist for the same provider, allowing users + to have different credentials for different environments (dev, prod, etc.). + + Attributes: + id: Unique identifier for this configuration + name: Human-readable name for this configuration + provider: Provider name (e.g., "openai", "anthropic") + is_default: Whether this is the default configuration for the provider + api_key: The API key (stored as SecretStr for in-memory protection) + base_url: Base URL for the provider API + model: Default model to use for this provider + api_version: API version string (for providers that need it) + endpoint: Generic endpoint URL + endpoint_llm: Endpoint URL for LLM service + endpoint_embedding: Endpoint URL for embedding service + endpoint_stt: Endpoint URL for speech-to-text service + endpoint_tts: Endpoint URL for text-to-speech service + project: Project ID (for Vertex AI) + location: Location/region (for Vertex AI) + credentials_path: Path to credentials file (for Vertex AI) + created: Timestamp when this config was created + updated: Timestamp when this config was last updated + """ + + def __init__( + self, + id: str, + name: str, + provider: str, + is_default: bool = False, + api_key: Optional[SecretStr] = None, + base_url: Optional[str] = None, + model: Optional[str] = None, + api_version: Optional[str] = None, + endpoint: Optional[str] = None, + endpoint_llm: Optional[str] = None, + endpoint_embedding: Optional[str] = None, + endpoint_stt: Optional[str] = None, + endpoint_tts: Optional[str] = None, + project: Optional[str] = None, + location: Optional[str] = None, + credentials_path: Optional[str] = None, + created: Optional[str] = None, + updated: Optional[str] = None, + ): + self.id = id + self.name = name + self.provider = provider + self.is_default = is_default + self.api_key = api_key + self.base_url = base_url + self.model = model + self.api_version = api_version + self.endpoint = endpoint + self.endpoint_llm = endpoint_llm + self.endpoint_embedding = endpoint_embedding + self.endpoint_stt = endpoint_stt + self.endpoint_tts = endpoint_tts + self.project = project + self.location = location + self.credentials_path = credentials_path + self.created = created or datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self.updated = updated or datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + def to_dict(self, encrypted: bool = False) -> dict: + """ + Convert the credential to a dictionary for storage. + + Args: + encrypted: If True, api_key is encrypted; otherwise it's a SecretStr + + Returns: + Dictionary representation of the credential + """ + data = { + "id": self.id, + "name": self.name, + "provider": self.provider, + "is_default": self.is_default, + "base_url": self.base_url, + "model": self.model, + "api_version": self.api_version, + "endpoint": self.endpoint, + "endpoint_llm": self.endpoint_llm, + "endpoint_embedding": self.endpoint_embedding, + "endpoint_stt": self.endpoint_stt, + "endpoint_tts": self.endpoint_tts, + "project": self.project, + "location": self.location, + "credentials_path": self.credentials_path, + "created": self.created, + "updated": self.updated, + } + + if self.api_key: + if encrypted: + data["api_key"] = encrypt_value(self.api_key.get_secret_value()) + else: + data["api_key"] = self.api_key.get_secret_value() + + return data + + @classmethod + def from_dict(cls, data: dict, decrypted: bool = False) -> "ProviderCredential": + """ + Create a ProviderCredential from a dictionary. + + Args: + data: Dictionary containing credential data + decrypted: If True, api_key is already decrypted; otherwise wrap in SecretStr + + Returns: + ProviderCredential instance + """ + api_key = None + if "api_key" in data and data["api_key"]: + if isinstance(data["api_key"], SecretStr): + # Already a SecretStr - use as-is + api_key = data["api_key"] + elif decrypted: + # Decrypted string from DB - wrap in SecretStr + api_key = SecretStr(data["api_key"]) + else: + # Encrypted string from DB - wrap in SecretStr (will be decrypted later) + api_key = SecretStr(data["api_key"]) + + return cls( + id=data["id"], + name=data["name"], + provider=data["provider"], + is_default=data.get("is_default", False), + api_key=api_key, + base_url=data.get("base_url"), + model=data.get("model"), + api_version=data.get("api_version"), + endpoint=data.get("endpoint"), + endpoint_llm=data.get("endpoint_llm"), + endpoint_embedding=data.get("endpoint_embedding"), + endpoint_stt=data.get("endpoint_stt"), + endpoint_tts=data.get("endpoint_tts"), + project=data.get("project"), + location=data.get("location"), + credentials_path=data.get("credentials_path"), + created=data.get("created"), + updated=data.get("updated"), + ) + + +class ProviderConfig(RecordModel): + """ + Singleton configuration for multiple provider credentials. + + Uses RecordModel pattern with a fixed record_id. Stores a dictionary + of ProviderCredential objects organized by provider name. + + Usage: + config = await ProviderConfig.get_instance() + credentials = config.credentials.get("openai", []) + default = config.get_default_config("openai") + """ + + record_id: ClassVar[str] = "open_notebook:provider_configs" + + # Store credentials organized by provider name + # Structure: {"openai": [ProviderCredential, ...], "anthropic": [...], ...} + credentials: Dict[str, List[ProviderCredential]] = Field( + default_factory=dict, + description="Provider credentials organized by provider name", + ) + + @classmethod + async def get_instance(cls) -> "ProviderConfig": + """ + Always fetch fresh configuration from database. + + Overrides parent caching behavior to ensure we always get the latest + configuration values. + + Returns: + ProviderConfig: Fresh instance with current database values + """ + result = await repo_query( + "SELECT * FROM ONLY $record_id", + {"record_id": ensure_record_id(cls.record_id)}, + ) + + if result: + if isinstance(result, list) and len(result) > 0: + data = result[0] + elif isinstance(result, dict): + data = result + else: + data = {} + else: + data = {} + + # Initialize credentials from database data + credentials: Dict[str, List[ProviderCredential]] = {} + creds_data = data.get("credentials") + if creds_data and isinstance(creds_data, dict): + for provider, provider_creds in creds_data.items(): + if isinstance(provider_creds, list): + credentials[provider] = [] + for cred_data in provider_creds: + try: + # Decrypt api_key if it's a string + api_key_val = cred_data.get("api_key") + if api_key_val and isinstance(api_key_val, str): + decrypted = decrypt_value(api_key_val) + cred_data["api_key"] = SecretStr(decrypted) + else: + # Keep as SecretStr or None + if api_key_val: + cred_data["api_key"] = SecretStr(api_key_val) + else: + cred_data["api_key"] = None + + credentials[provider].append( + ProviderCredential( + id=cred_data.get("id", ""), + name=cred_data.get("name", "Default"), + provider=cred_data.get("provider", provider), + is_default=cred_data.get("is_default", False), + api_key=cred_data.get("api_key"), + base_url=cred_data.get("base_url"), + model=cred_data.get("model"), + api_version=cred_data.get("api_version"), + endpoint=cred_data.get("endpoint"), + endpoint_llm=cred_data.get("endpoint_llm"), + endpoint_embedding=cred_data.get( + "endpoint_embedding" + ), + endpoint_stt=cred_data.get("endpoint_stt"), + endpoint_tts=cred_data.get("endpoint_tts"), + project=cred_data.get("project"), + location=cred_data.get("location"), + credentials_path=cred_data.get("credentials_path"), + created=cred_data.get("created"), + updated=cred_data.get("updated"), + ) + ) + except Exception: + # Skip invalid credentials + continue + + # Create instance using model_validate to properly initialize Pydantic model + instance = cls.model_validate({"credentials": credentials}) + + # Mark as loaded from database + object.__setattr__(instance, "_db_loaded", True) + + return instance + + def get_default_config(self, provider: str) -> Optional[ProviderCredential]: + """ + Get the default configuration for a provider. + + Args: + provider: Provider name (e.g., "openai", "anthropic") + + Returns: + The default ProviderCredential, or None if not found + """ + provider_lower = provider.lower() + credentials = self.credentials.get(provider_lower, []) + + # First, try to find explicitly marked default + for cred in credentials: + if cred.is_default: + return cred + + # If no explicit default, return first config + if credentials: + return credentials[0] + + return None + + def get_config( + self, provider: str, config_id: str + ) -> Optional[ProviderCredential]: + """ + Get a specific configuration by ID. + + Args: + provider: Provider name + config_id: Configuration ID + + Returns: + The ProviderCredential if found, None otherwise + """ + provider_lower = provider.lower() + credentials = self.credentials.get(provider_lower, []) + + for cred in credentials: + if cred.id == config_id: + return cred + + return None + + def add_config(self, provider: str, credential: ProviderCredential) -> None: + """ + Add a new configuration for a provider. + + If this is the first config for the provider, it becomes the default. + When adding a new config to an existing provider, the new config becomes + the default and previous default is unset. + + Args: + provider: Provider name (normalized to lowercase) + credential: ProviderCredential to add + """ + provider_lower = provider.lower() + credential.provider = provider_lower + + if provider_lower not in self.credentials: + self.credentials[provider_lower] = [] + + # When adding a new config to an existing provider, make it the default + # and unset the previous default + if self.credentials[provider_lower]: + for cred in self.credentials[provider_lower]: + cred.is_default = False + credential.is_default = True + + # If this is the first config, make it default + if not self.credentials[provider_lower]: + credential.is_default = True + + self.credentials[provider_lower].append(credential) + + def delete_config(self, provider: str, config_id: str) -> bool: + """ + Delete a configuration. + + Cannot delete the default configuration unless it's the only one. + + Args: + provider: Provider name + config_id: Configuration ID to delete + + Returns: + True if deleted, False if not found + """ + provider_lower = provider.lower() + credentials = self.credentials.get(provider_lower, []) + + for i, cred in enumerate(credentials): + if cred.id == config_id: + # Cannot delete default if there are other configs + if cred.is_default and len(credentials) > 1: + return False + + del credentials[i] + return True + + return False + + def set_default_config(self, provider: str, config_id: str) -> bool: + """ + Set a configuration as the default for a provider. + + Args: + provider: Provider name + config_id: Configuration ID to make default + + Returns: + True if successful, False if config not found + """ + provider_lower = provider.lower() + credentials = self.credentials.get(provider_lower, []) + + for cred in credentials: + if cred.id == config_id: + # Unset all other defaults + for other in credentials: + other.is_default = False + + # Set this one as default + cred.is_default = True + cred.updated = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return True + + return False + + def _prepare_save_data(self) -> dict: + """ + Prepare data for database storage. + + SecretStr values are extracted, encrypted, and stored as strings. + Encryption is performed using Fernet symmetric encryption if + OPEN_NOTEBOOK_ENCRYPTION_KEY is configured. + """ + data = {"credentials": {}} + + for provider, credentials in self.credentials.items(): + data["credentials"][provider] = [] + for cred in credentials: + cred.updated = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + data["credentials"][provider].append(cred.to_dict(encrypted=True)) + + return data + + async def save(self) -> "ProviderConfig": + """ + Save the configuration to the database. + + Uses _prepare_save_data() to properly handle SecretStr conversion + and encryption. + """ + data = self._prepare_save_data() + await repo_upsert("open_notebook", self.record_id, data) + return self + + @classmethod + def _clear_for_test(cls) -> None: + """Clear the singleton instance for testing purposes.""" + if cls.record_id in cls._instances: + del cls._instances[cls.record_id] diff --git a/open_notebook/domain/transformation.py b/open_notebook/domain/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..4e979767039a0acba25b196da0866dbaaeb2f076 --- /dev/null +++ b/open_notebook/domain/transformation.py @@ -0,0 +1,21 @@ +from typing import ClassVar, Optional + +from pydantic import Field + +from open_notebook.domain.base import ObjectModel, RecordModel + + +class Transformation(ObjectModel): + table_name: ClassVar[str] = "transformation" + name: str + title: str + description: str + prompt: str + apply_default: bool + + +class DefaultPrompts(RecordModel): + record_id: ClassVar[str] = "open_notebook:default_prompts" + transformation_instructions: Optional[str] = Field( + None, description="Instructions for executing a transformation" + ) diff --git a/open_notebook/exceptions.py b/open_notebook/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..49be004f92c292b46058d19e777c72dd88097b72 --- /dev/null +++ b/open_notebook/exceptions.py @@ -0,0 +1,70 @@ +class OpenNotebookError(Exception): + """Base exception class for Open Notebook errors.""" + + pass + + +class DatabaseOperationError(OpenNotebookError): + """Raised when a database operation fails.""" + + pass + + +class UnsupportedTypeException(OpenNotebookError): + """Raised when an unsupported type is provided.""" + + pass + + +class InvalidInputError(OpenNotebookError): + """Raised when invalid input is provided.""" + + pass + + +class NotFoundError(OpenNotebookError): + """Raised when a requested resource is not found.""" + + pass + + +class AuthenticationError(OpenNotebookError): + """Raised when there's an authentication problem.""" + + pass + + +class ConfigurationError(OpenNotebookError): + """Raised when there's a configuration problem.""" + + pass + + +class ExternalServiceError(OpenNotebookError): + """Raised when an external service (e.g., AI model) fails.""" + + pass + + +class RateLimitError(OpenNotebookError): + """Raised when a rate limit is exceeded.""" + + pass + + +class FileOperationError(OpenNotebookError): + """Raised when a file operation fails.""" + + pass + + +class NetworkError(OpenNotebookError): + """Raised when a network operation fails.""" + + pass + + +class NoTranscriptFound(OpenNotebookError): + """Raised when no transcript is found for a video.""" + + pass diff --git a/open_notebook/graphs/CLAUDE.md b/open_notebook/graphs/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..1b4660aa170003330ad31b2a25ee65166fc78f7e --- /dev/null +++ b/open_notebook/graphs/CLAUDE.md @@ -0,0 +1,79 @@ +# Graphs Module + +LangGraph-based workflow orchestration for content processing, chat interactions, and AI-powered transformations. + +## Key Components + +- **`chat.py`**: Conversational agent with message history, notebook context, and model override support +- **`source_chat.py`**: Source-focused chat with ContextBuilder for insights/content injection and context tracking +- **`ask.py`**: Multi-search strategy agent (generates search terms, retrieves results, synthesizes answers) +- **`source.py`**: Content ingestion pipeline (extract → save → transform with content-core) +- **`transformation.py`**: Single-node transformation executor with prompt templating via ai_prompter +- **`prompt.py`**: Generic pattern chain for arbitrary prompt-based LLM calls +- **`tools.py`**: Minimal tool library (currently just `get_current_timestamp()`) + +## Important Patterns + +- **Async/sync bridging in graphs**: Both `chat.py` and `source_chat.py` use `asyncio.new_event_loop()` workaround because LangGraph nodes are sync but `provision_langchain_model()` is async +- **State machines via StateGraph**: Each graph compiles to stateful runnable; conditional edges fan out work (ask.py, source.py do parallel transforms) +- **Prompt templating**: `ai_prompter.Prompter` with Jinja2 templates referenced by path ("chat/system", "ask/entry", etc.) +- **Model provisioning via context**: Config dict passed to node via `RunnableConfig`; defaults fall back to state overrides +- **Checkpointing**: `chat.py` and `source_chat.py` use SqliteSaver for message history (LangGraph's built-in persistence) +- **Content extraction**: `source.py` uses content-core library with provider/model from DefaultModels; URLs and files both supported + +## Error Handling in Graphs + +All graph nodes use `classify_error()` from `open_notebook.utils.error_classifier` to catch raw LLM provider exceptions and re-raise them as typed `OpenNotebookError` subclasses with user-friendly messages. This ensures that errors from any AI provider (authentication failures, rate limits, model not found, network issues) are surfaced to the user with actionable messages instead of opaque stack traces. + +**Pattern in nodes**: +```python +from open_notebook.utils.error_classifier import classify_error + +try: + result = await model.ainvoke(...) +except Exception as e: + exc_class, message = classify_error(e) + raise exc_class(message) from e +``` + +--- + +## Quirks & Edge Cases + +- **Async loop gymnastics**: ThreadPoolExecutor workaround needed because LangGraph invokes sync nodes but we call async functions; fragile if event loop state changes +- **`clean_thinking_content()` ubiquitous**: Strips `...` tags from model responses (handles extended thinking models) +- **source_chat.py builds context twice**: ContextBuilder runs during node execution to fetch source/insights; rebuilds list from context_data (inefficient but safe) +- **source.py embedding is async**: `source.vectorize()` returns job command ID; not awaited (fire-and-forget) +- **transformation.py nullable source**: Accepts `input_text` or `source.full_text` (falls back to second if first missing) +- **ask.py hard-coded vector_search**: No fallback to text search despite commented code suggesting it was planned +- **SqliteSaver location**: Checkpoints stored in path from `LANGGRAPH_CHECKPOINT_FILE` env var; connection shared across graphs + +## Key Dependencies + +- `langgraph`: StateGraph, Send, END, START, SqliteSaver checkpoint persistence +- `langchain_core`: Messages, OutputParser, RunnableConfig +- `ai_prompter`: Prompter for Jinja2 template rendering +- `content_core`: `extract_content()` for file/URL processing +- `open_notebook.ai.provision`: `provision_langchain_model()` (async factory with fallback logic) +- `open_notebook.utils.error_classifier`: `classify_error()` for user-friendly LLM error messages +- `open_notebook.domain.notebook`: Domain models (Source, Note, SourceInsight, vector_search) +- `loguru`: Logging + +## Usage Example + +```python +# Invoke a graph with config override +config = {"configurable": {"model_id": "model:custom_id"}} +result = await chat_graph.ainvoke( + {"messages": [HumanMessage(content="...")], "notebook": notebook}, + config=config +) + +# Source processing (content → save → transform) +result = await source_graph.ainvoke({ + "content_state": {...}, # ProcessSourceState from content-core + "apply_transformations": [t1, t2], + "source_id": "source:123", + "embed": True +}) +``` diff --git a/open_notebook/graphs/ask.py b/open_notebook/graphs/ask.py new file mode 100644 index 0000000000000000000000000000000000000000..74f0ae6b94a41e3224c61e9b609a01120fbd7956 --- /dev/null +++ b/open_notebook/graphs/ask.py @@ -0,0 +1,155 @@ +import operator +from typing import Annotated, List + +from ai_prompter import Prompter +from langchain_core.output_parsers.pydantic import PydanticOutputParser +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, START, StateGraph +from langgraph.types import Send +from pydantic import BaseModel, Field +from typing_extensions import TypedDict + +from open_notebook.ai.provision import provision_langchain_model +from open_notebook.domain.notebook import vector_search +from open_notebook.exceptions import OpenNotebookError +from open_notebook.utils import clean_thinking_content +from open_notebook.utils.error_classifier import classify_error +from open_notebook.utils.text_utils import extract_text_content + + +class SubGraphState(TypedDict): + question: str + term: str + instructions: str + results: dict + answer: str + ids: list # Added for provide_answer function + + +class Search(BaseModel): + term: str + instructions: str = Field( + description="Tell the answeting LLM what information you need extracted from this search" + ) + + +class Strategy(BaseModel): + reasoning: str + searches: List[Search] = Field( + default_factory=list, + description="You can add up to five searches to this strategy", + ) + + +class ThreadState(TypedDict): + question: str + strategy: Strategy + answers: Annotated[list, operator.add] + final_answer: str + + +async def call_model_with_messages(state: ThreadState, config: RunnableConfig) -> dict: + try: + parser = PydanticOutputParser(pydantic_object=Strategy) + system_prompt = Prompter(prompt_template="ask/entry", parser=parser).render( # type: ignore[arg-type] + data=state # type: ignore[arg-type] + ) + model = await provision_langchain_model( + system_prompt, + config.get("configurable", {}).get("strategy_model"), + "tools", + max_tokens=2000, + structured=dict(type="json"), + ) + # model = model.bind_tools(tools) + # First get the raw response from the model + ai_message = await model.ainvoke(system_prompt) + + # Clean the thinking content from the response + message_content = extract_text_content(ai_message.content) + cleaned_content = clean_thinking_content(message_content) + + # Parse the cleaned JSON content + strategy = parser.parse(cleaned_content) + + return {"strategy": strategy} + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +async def trigger_queries(state: ThreadState, config: RunnableConfig): + return [ + Send( + "provide_answer", + { + "question": state["question"], + "instructions": s.instructions, + "term": s.term, + # "type": s.type, + }, + ) + for s in state["strategy"].searches + ] + + +async def provide_answer(state: SubGraphState, config: RunnableConfig) -> dict: + try: + payload = state + # if state["type"] == "text": + # results = text_search(state["term"], 10, True, True) + # else: + results = await vector_search(state["term"], 10, True, True) + if len(results) == 0: + return {"answers": []} + payload["results"] = results + ids = [r["id"] for r in results] + payload["ids"] = ids + system_prompt = Prompter(prompt_template="ask/query_process").render(data=payload) # type: ignore[arg-type] + model = await provision_langchain_model( + system_prompt, + config.get("configurable", {}).get("answer_model"), + "tools", + max_tokens=2000, + ) + ai_message = await model.ainvoke(system_prompt) + ai_content = extract_text_content(ai_message.content) + return {"answers": [clean_thinking_content(ai_content)]} + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +async def write_final_answer(state: ThreadState, config: RunnableConfig) -> dict: + try: + system_prompt = Prompter(prompt_template="ask/final_answer").render(data=state) # type: ignore[arg-type] + model = await provision_langchain_model( + system_prompt, + config.get("configurable", {}).get("final_answer_model"), + "tools", + max_tokens=2000, + ) + ai_message = await model.ainvoke(system_prompt) + final_content = extract_text_content(ai_message.content) + return {"final_answer": clean_thinking_content(final_content)} + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +agent_state = StateGraph(ThreadState) +agent_state.add_node("agent", call_model_with_messages) +agent_state.add_node("provide_answer", provide_answer) +agent_state.add_node("write_final_answer", write_final_answer) +agent_state.add_edge(START, "agent") +agent_state.add_conditional_edges("agent", trigger_queries, ["provide_answer"]) +agent_state.add_edge("provide_answer", "write_final_answer") +agent_state.add_edge("write_final_answer", END) + +graph = agent_state.compile() diff --git a/open_notebook/graphs/chat.py b/open_notebook/graphs/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..dced3989cd417492b2cfddd571b9d7885dec5129 --- /dev/null +++ b/open_notebook/graphs/chat.py @@ -0,0 +1,98 @@ +import asyncio +import sqlite3 +from typing import Annotated, Optional + +from ai_prompter import Prompter +from langchain_core.messages import AIMessage, SystemMessage +from langchain_core.runnables import RunnableConfig +from langgraph.checkpoint.sqlite import SqliteSaver +from langgraph.graph import END, START, StateGraph +from langgraph.graph.message import add_messages +from typing_extensions import TypedDict + +from open_notebook.ai.provision import provision_langchain_model +from open_notebook.config import LANGGRAPH_CHECKPOINT_FILE +from open_notebook.domain.notebook import Notebook +from open_notebook.exceptions import OpenNotebookError +from open_notebook.utils import clean_thinking_content +from open_notebook.utils.error_classifier import classify_error +from open_notebook.utils.text_utils import extract_text_content + + +class ThreadState(TypedDict): + messages: Annotated[list, add_messages] + notebook: Optional[Notebook] + context: Optional[str] + context_config: Optional[dict] + model_override: Optional[str] + + +def call_model_with_messages(state: ThreadState, config: RunnableConfig) -> dict: + try: + system_prompt = Prompter(prompt_template="chat/system").render(data=state) # type: ignore[arg-type] + payload = [SystemMessage(content=system_prompt)] + state.get("messages", []) + model_id = config.get("configurable", {}).get("model_id") or state.get( + "model_override" + ) + + # Handle async model provisioning from sync context + def run_in_new_loop(): + """Run the async function in a new event loop""" + new_loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(new_loop) + return new_loop.run_until_complete( + provision_langchain_model( + str(payload), model_id, "chat", max_tokens=8192 + ) + ) + finally: + new_loop.close() + asyncio.set_event_loop(None) + + try: + # Try to get the current event loop + asyncio.get_running_loop() + # If we're in an event loop, run in a thread with a new loop + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(run_in_new_loop) + model = future.result() + except RuntimeError: + # No event loop running, safe to use asyncio.run() + model = asyncio.run( + provision_langchain_model( + str(payload), + model_id, + "chat", + max_tokens=8192, + ) + ) + + ai_message = model.invoke(payload) + + # Clean thinking content from AI response (e.g., ... tags) + content = extract_text_content(ai_message.content) + cleaned_content = clean_thinking_content(content) + cleaned_message = ai_message.model_copy(update={"content": cleaned_content}) + + return {"messages": cleaned_message} + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +conn = sqlite3.connect( + LANGGRAPH_CHECKPOINT_FILE, + check_same_thread=False, +) +memory = SqliteSaver(conn) + +agent_state = StateGraph(ThreadState) +agent_state.add_node("agent", call_model_with_messages) +agent_state.add_edge(START, "agent") +agent_state.add_edge("agent", END) +graph = agent_state.compile(checkpointer=memory) diff --git a/open_notebook/graphs/prompt.py b/open_notebook/graphs/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..cd80d10f45c59fff10e48b66b976b710bd68575d --- /dev/null +++ b/open_notebook/graphs/prompt.py @@ -0,0 +1,45 @@ +from typing import Any, Optional + +from ai_prompter import Prompter +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, START, StateGraph +from typing_extensions import TypedDict + +from open_notebook.ai.provision import provision_langchain_model +from open_notebook.utils.text_utils import clean_thinking_content, extract_text_content + + +class PatternChainState(TypedDict): + prompt: str + parser: Optional[Any] + input_text: str + output: str + + +async def call_model(state: dict, config: RunnableConfig) -> dict: + content = state["input_text"] + system_prompt = Prompter( + template_text=state["prompt"], parser=state.get("parser") + ).render(data=state) + payload = [SystemMessage(content=system_prompt)] + [HumanMessage(content=content)] + chain = await provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id"), + "transformation", + max_tokens=5000, + ) + + response = await chain.ainvoke(payload) + + # Clean thinking tags from response (handles extended thinking models) + output = clean_thinking_content(extract_text_content(response.content)) + return {"output": output} + + +agent_state = StateGraph(PatternChainState) +agent_state.add_node("agent", call_model) # type: ignore[type-var] +agent_state.add_edge(START, "agent") +agent_state.add_edge("agent", END) + +graph = agent_state.compile() diff --git a/open_notebook/graphs/source.py b/open_notebook/graphs/source.py new file mode 100644 index 0000000000000000000000000000000000000000..617396b7f58470a694ca4fc4ef111d8885d46c06 --- /dev/null +++ b/open_notebook/graphs/source.py @@ -0,0 +1,187 @@ +import operator +from typing import Any, Dict, List, Optional + +from content_core import extract_content +from content_core.common import ProcessSourceState +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, START, StateGraph +from langgraph.types import Send +from loguru import logger +from typing_extensions import Annotated, TypedDict + +from open_notebook.ai.models import Model, ModelManager +from open_notebook.domain.content_settings import ContentSettings +from open_notebook.domain.notebook import Asset, Source +from open_notebook.domain.transformation import Transformation +from open_notebook.graphs.transformation import graph as transform_graph + + +class SourceState(TypedDict): + content_state: ProcessSourceState + apply_transformations: List[Transformation] + source_id: str + notebook_ids: List[str] + source: Source + transformation: Annotated[list, operator.add] + embed: bool + + +class TransformationState(TypedDict): + source: Source + transformation: Transformation + + +async def content_process(state: SourceState) -> dict: + content_settings = ContentSettings( + default_content_processing_engine_doc="auto", + default_content_processing_engine_url="auto", + default_embedding_option="ask", + auto_delete_files="yes", + youtube_preferred_languages=[ + "en", + "pt", + "es", + "de", + "nl", + "en-GB", + "fr", + "hi", + "ja", + ], + ) + content_state: Dict[str, Any] = state["content_state"] # type: ignore[assignment] + + content_state["url_engine"] = ( + content_settings.default_content_processing_engine_url or "auto" + ) + content_state["document_engine"] = ( + content_settings.default_content_processing_engine_doc or "auto" + ) + content_state["output_format"] = "markdown" + + # Add speech-to-text model configuration from Default Models + try: + model_manager = ModelManager() + defaults = await model_manager.get_defaults() + if defaults.default_speech_to_text_model: + stt_model = await Model.get(defaults.default_speech_to_text_model) + if stt_model: + content_state["audio_provider"] = stt_model.provider + content_state["audio_model"] = stt_model.name + logger.debug( + f"Using speech-to-text model: {stt_model.provider}/{stt_model.name}" + ) + except Exception as e: + logger.warning(f"Failed to retrieve speech-to-text model configuration: {e}") + # Continue without custom audio model (content-core will use its default) + + processed_state = await extract_content(content_state) + + if not processed_state.content or not processed_state.content.strip(): + url = processed_state.url or "" + if url and ("youtube.com" in url or "youtu.be" in url): + raise ValueError( + "Could not extract content from this YouTube video. " + "No transcript or subtitles are available. " + "Try configuring a Speech-to-Text model in Settings " + "to transcribe the audio instead." + ) + raise ValueError( + "Could not extract any text content from this source. " + "The content may be empty, inaccessible, or in an unsupported format." + ) + + return {"content_state": processed_state} + + +async def save_source(state: SourceState) -> dict: + content_state = state["content_state"] + + # Get existing source using the provided source_id + source = await Source.get(state["source_id"]) + if not source: + raise ValueError(f"Source with ID {state['source_id']} not found") + + # Update the source with processed content + source.asset = Asset(url=content_state.url, file_path=content_state.file_path) + source.full_text = content_state.content + + # Preserve user-set title; only overwrite placeholder or empty titles + if content_state.title and (not source.title or source.title == "Processing..."): + source.title = content_state.title + + await source.save() + + # NOTE: Notebook associations are created by the API immediately for UI responsiveness + # No need to create them here to avoid duplicate edges + + if state["embed"]: + if source.full_text and source.full_text.strip(): + logger.debug("Embedding content for vector search") + await source.vectorize() + else: + logger.warning( + f"Source {source.id} has no text content to embed, skipping vectorization" + ) + + return {"source": source} + + +def trigger_transformations(state: SourceState, config: RunnableConfig) -> List[Send]: + if len(state["apply_transformations"]) == 0: + return [] + + to_apply = state["apply_transformations"] + logger.debug(f"Applying transformations {to_apply}") + + return [ + Send( + "transform_content", + { + "source": state["source"], + "transformation": t, + }, + ) + for t in to_apply + ] + + +async def transform_content(state: TransformationState) -> Optional[dict]: + source = state["source"] + content = source.full_text + if not content: + return None + transformation: Transformation = state["transformation"] + + logger.debug(f"Applying transformation {transformation.name}") + result = await transform_graph.ainvoke( + dict(input_text=content, transformation=transformation) # type: ignore[arg-type] + ) + await source.add_insight(transformation.title, result["output"]) + return { + "transformation": [ + { + "output": result["output"], + "transformation_name": transformation.name, + } + ] + } + + +# Create and compile the workflow +workflow = StateGraph(SourceState) + +# Add nodes +workflow.add_node("content_process", content_process) +workflow.add_node("save_source", save_source) +workflow.add_node("transform_content", transform_content) +# Define the graph edges +workflow.add_edge(START, "content_process") +workflow.add_edge("content_process", "save_source") +workflow.add_conditional_edges( + "save_source", trigger_transformations, ["transform_content"] +) +workflow.add_edge("transform_content", END) + +# Compile the graph +source_graph = workflow.compile() diff --git a/open_notebook/graphs/source_chat.py b/open_notebook/graphs/source_chat.py new file mode 100644 index 0000000000000000000000000000000000000000..bccc25e8ad3392bd21ef6d2d3e99a4437cf0ef48 --- /dev/null +++ b/open_notebook/graphs/source_chat.py @@ -0,0 +1,255 @@ +import asyncio +import sqlite3 +from typing import Annotated, Dict, List, Optional + +from ai_prompter import Prompter +from langchain_core.messages import AIMessage, SystemMessage +from langchain_core.runnables import RunnableConfig +from langgraph.checkpoint.sqlite import SqliteSaver +from langgraph.graph import END, START, StateGraph +from langgraph.graph.message import add_messages +from typing_extensions import TypedDict + +from open_notebook.ai.provision import provision_langchain_model +from open_notebook.config import LANGGRAPH_CHECKPOINT_FILE +from open_notebook.domain.notebook import Source, SourceInsight +from open_notebook.exceptions import OpenNotebookError +from open_notebook.utils import clean_thinking_content +from open_notebook.utils.context_builder import ContextBuilder +from open_notebook.utils.error_classifier import classify_error +from open_notebook.utils.text_utils import extract_text_content + + +class SourceChatState(TypedDict): + messages: Annotated[list, add_messages] + source_id: str + source: Optional[Source] + insights: Optional[List[SourceInsight]] + context: Optional[str] + model_override: Optional[str] + context_indicators: Optional[Dict[str, List[str]]] + + +def call_model_with_source_context( + state: SourceChatState, config: RunnableConfig +) -> dict: + """ + Main function that builds source context and calls the model. + + This function: + 1. Uses ContextBuilder to build source-specific context + 2. Applies the source_chat Jinja2 prompt template + 3. Handles model provisioning with override support + 4. Tracks context indicators for referenced insights/content + """ + try: + return _call_model_with_source_context_inner(state, config) + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +def _call_model_with_source_context_inner( + state: SourceChatState, config: RunnableConfig +) -> dict: + source_id = state.get("source_id") + if not source_id: + raise ValueError("source_id is required in state") + + # Build source context using ContextBuilder (run async code in new loop) + def build_context(): + """Build context in a new event loop""" + new_loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(new_loop) + context_builder = ContextBuilder( + source_id=source_id, + include_insights=True, + include_notes=False, # Focus on source-specific content + max_tokens=50000, # Reasonable limit for source context + ) + return new_loop.run_until_complete(context_builder.build()) + finally: + new_loop.close() + asyncio.set_event_loop(None) + + # Get the built context + try: + # Try to get the current event loop + asyncio.get_running_loop() + # If we're in an event loop, run in a thread with a new loop + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(build_context) + context_data = future.result() + except RuntimeError: + # No event loop running, safe to create a new one + context_data = build_context() + + # Extract source and insights from context + source = None + insights = [] + context_indicators: dict[str, list[str | None]] = { + "sources": [], + "insights": [], + "notes": [], + } + + if context_data.get("sources"): + source_info = context_data["sources"][0] # First source + source = Source(**source_info) if isinstance(source_info, dict) else source_info + context_indicators["sources"].append(source.id) + + if context_data.get("insights"): + for insight_data in context_data["insights"]: + insight = ( + SourceInsight(**insight_data) + if isinstance(insight_data, dict) + else insight_data + ) + insights.append(insight) + context_indicators["insights"].append(insight.id) + + # Format context for the prompt + formatted_context = _format_source_context(context_data) + + # Build prompt data for the template + prompt_data = { + "source": source.model_dump() if source else None, + "insights": [insight.model_dump() for insight in insights] if insights else [], + "context": formatted_context, + "context_indicators": context_indicators, + } + + # Apply the source_chat prompt template + system_prompt = Prompter(prompt_template="source_chat/system").render( + data=prompt_data + ) + payload = [SystemMessage(content=system_prompt)] + state.get("messages", []) + + # Handle async model provisioning from sync context + def run_in_new_loop(): + """Run the async function in a new event loop""" + new_loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(new_loop) + return new_loop.run_until_complete( + provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id") + or state.get("model_override"), + "chat", + max_tokens=8192, + ) + ) + finally: + new_loop.close() + asyncio.set_event_loop(None) + + try: + # Try to get the current event loop + asyncio.get_running_loop() + # If we're in an event loop, run in a thread with a new loop + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(run_in_new_loop) + model = future.result() + except RuntimeError: + # No event loop running, safe to use asyncio.run() + model = asyncio.run( + provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id") + or state.get("model_override"), + "chat", + max_tokens=8192, + ) + ) + + ai_message = model.invoke(payload) + + # Clean thinking content from AI response (e.g., ... tags) + content = extract_text_content(ai_message.content) + cleaned_content = clean_thinking_content(content) + cleaned_message = ai_message.model_copy(update={"content": cleaned_content}) + + # Update state with context information + return { + "messages": cleaned_message, + "source": source, + "insights": insights, + "context": formatted_context, + "context_indicators": context_indicators, + } + + +def _format_source_context(context_data: Dict) -> str: + """ + Format the context data into a readable string for the prompt. + + Args: + context_data: Context data from ContextBuilder + + Returns: + Formatted context string + """ + context_parts = [] + + # Add source information + if context_data.get("sources"): + context_parts.append("## SOURCE CONTENT") + for source in context_data["sources"]: + if isinstance(source, dict): + context_parts.append(f"**Source ID:** {source.get('id', 'Unknown')}") + context_parts.append(f"**Title:** {source.get('title', 'No title')}") + if source.get("full_text"): + # Truncate full text if too long + full_text = source["full_text"] + if len(full_text) > 5000: + full_text = full_text[:5000] + "...\n[Content truncated]" + context_parts.append(f"**Content:**\n{full_text}") + context_parts.append("") # Empty line for separation + + # Add insights + if context_data.get("insights"): + context_parts.append("## SOURCE INSIGHTS") + for insight in context_data["insights"]: + if isinstance(insight, dict): + context_parts.append(f"**Insight ID:** {insight.get('id', 'Unknown')}") + context_parts.append( + f"**Type:** {insight.get('insight_type', 'Unknown')}" + ) + context_parts.append( + f"**Content:** {insight.get('content', 'No content')}" + ) + context_parts.append("") # Empty line for separation + + # Add metadata + if context_data.get("metadata"): + metadata = context_data["metadata"] + context_parts.append("## CONTEXT METADATA") + context_parts.append(f"- Source count: {metadata.get('source_count', 0)}") + context_parts.append(f"- Insight count: {metadata.get('insight_count', 0)}") + context_parts.append(f"- Total tokens: {context_data.get('total_tokens', 0)}") + context_parts.append("") + + return "\n".join(context_parts) + + +# Create SQLite checkpointer +conn = sqlite3.connect( + LANGGRAPH_CHECKPOINT_FILE, + check_same_thread=False, +) +memory = SqliteSaver(conn) + +# Create the StateGraph +source_chat_state = StateGraph(SourceChatState) +source_chat_state.add_node("source_chat_agent", call_model_with_source_context) +source_chat_state.add_edge(START, "source_chat_agent") +source_chat_state.add_edge("source_chat_agent", END) +source_chat_graph = source_chat_state.compile(checkpointer=memory) diff --git a/open_notebook/graphs/tools.py b/open_notebook/graphs/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..9c3df13b82ceb76a6cc418a13b255203bcebbdc9 --- /dev/null +++ b/open_notebook/graphs/tools.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from langchain.tools import tool + + +# todo: turn this into a system prompt variable +@tool +def get_current_timestamp() -> str: + """ + name: get_current_timestamp + Returns the current timestamp in the format YYYYMMDDHHmmss. + """ + return datetime.now().strftime("%Y%m%d%H%M%S") diff --git a/open_notebook/graphs/transformation.py b/open_notebook/graphs/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..65bc5267ec98ca19772f88b14938fd20f06ed9eb --- /dev/null +++ b/open_notebook/graphs/transformation.py @@ -0,0 +1,75 @@ +from ai_prompter import Prompter +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, START, StateGraph +from typing_extensions import TypedDict + +from open_notebook.ai.provision import provision_langchain_model +from open_notebook.domain.notebook import Source +from open_notebook.domain.transformation import DefaultPrompts, Transformation +from open_notebook.exceptions import OpenNotebookError +from open_notebook.utils import clean_thinking_content +from open_notebook.utils.error_classifier import classify_error +from open_notebook.utils.text_utils import extract_text_content + + +class TransformationState(TypedDict): + input_text: str + source: Source + transformation: Transformation + output: str + + +async def run_transformation(state: dict, config: RunnableConfig) -> dict: + source_obj = state.get("source") + source: Source = source_obj if isinstance(source_obj, Source) else None # type: ignore[assignment] + content = state.get("input_text") + assert source or content, "No content to transform" + transformation: Transformation = state["transformation"] + + try: + if not content: + content = source.full_text + transformation_template_text = transformation.prompt + default_prompts: DefaultPrompts = DefaultPrompts(transformation_instructions=None) + if default_prompts.transformation_instructions: + transformation_template_text = f"{default_prompts.transformation_instructions}\n\n{transformation_template_text}" + + transformation_template_text = f"{transformation_template_text}\n\n# INPUT" + + system_prompt = Prompter(template_text=transformation_template_text).render( + data=state + ) + content_str = str(content) if content else "" + payload = [SystemMessage(content=system_prompt), HumanMessage(content=content_str)] + chain = await provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id"), + "transformation", + max_tokens=8192, + ) + + response = await chain.ainvoke(payload) + + # Clean thinking content from the response + response_content = extract_text_content(response.content) + cleaned_content = clean_thinking_content(response_content) + + if source: + await source.add_insight(transformation.title, cleaned_content) + + return { + "output": cleaned_content, + } + except OpenNotebookError: + raise + except Exception as e: + error_class, user_message = classify_error(e) + raise error_class(user_message) from e + + +agent_state = StateGraph(TransformationState) +agent_state.add_node("agent", run_transformation) # type: ignore[type-var] +agent_state.add_edge(START, "agent") +agent_state.add_edge("agent", END) +graph = agent_state.compile() diff --git a/open_notebook/podcasts/CLAUDE.md b/open_notebook/podcasts/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..f2f225acf9203177339eb6da15f5d24de1506990 --- /dev/null +++ b/open_notebook/podcasts/CLAUDE.md @@ -0,0 +1,103 @@ +# Podcasts Module + +Domain models for podcast generation featuring speaker and episode profile management with job tracking. + +## Purpose + +Encapsulates podcast metadata and configuration: speaker profiles (voice/personality config), episode profiles (generation settings), and podcast episodes (with job status tracking via surreal-commands). + +## Architecture Overview + +Two-tier profile system using the **model registry** for AI model references: +- **SpeakerProfile**: `voice_model` (record reference) + 1-4 speaker configurations (name, voice_id, backstory, personality). Per-speaker `voice_model` overrides supported. +- **EpisodeProfile**: `outline_llm`/`transcript_llm` (record references) for LLM selection, `language` field (BCP 47 locale code), segment count, briefing template. +- **PodcastEpisode**: Generated episode record linking profiles, content, and async job. + +All inherit from `ObjectModel` (SurrealDB base class with table_name and save/load). + +## Component Catalog + +### models.py + +#### `_resolve_model_config(model_id)` (module-level helper) +- Loads a Model record by ID, resolves its credential, returns `(provider, model_name, config_dict)` tuple. +- Used by `resolve_outline_config()`, `resolve_transcript_config()`, `resolve_tts_config()`, and per-speaker TTS overrides in `podcast_commands.py`. +- Falls back to `provision_provider_keys()` if no credential is linked. + +#### SpeakerProfile +- `voice_model`: Optional `record` reference for TTS (replaces legacy `tts_provider`/`tts_model` strings). +- Legacy fields `tts_provider`/`tts_model` kept as optional for migration compatibility. +- `nullable_fields` ClassVar lists fields that may be null in the database. +- Validates 1-4 speakers with required fields: name, voice_id, backstory, personality. +- Per-speaker `voice_model` override: individual speakers can reference a different TTS model. +- `_prepare_save_data()` converts `voice_model` (and per-speaker overrides) to RecordID before save. +- `resolve_tts_config()` resolves `voice_model` via `_resolve_model_config()`. Raises ValueError if not set. +- `get_by_name()` async query by profile name. + +#### EpisodeProfile +- `outline_llm`/`transcript_llm`: Optional `record` references (replace legacy `outline_provider`/`outline_model`/`transcript_provider`/`transcript_model` strings). +- `language`: Optional BCP 47 locale code for podcast language (e.g. `pt-BR`, `en-US`). +- Legacy fields kept as optional for migration compatibility. +- `nullable_fields` ClassVar lists fields that may be null in the database. +- `num_segments` validated between 3 and 20. +- References `speaker_config` by name. +- `_prepare_save_data()` converts `outline_llm`/`transcript_llm` to RecordID before save. +- `resolve_outline_config()` / `resolve_transcript_config()` resolve model references via `_resolve_model_config()`. Raise ValueError if not set. +- `get_by_name()` async query. + +#### PodcastEpisode +- Stores episode_profile and speaker_profile as dicts (snapshots of config at generation time). +- Optional audio_file path, transcript/outline dicts. +- **Job tracking**: command field links to surreal-commands RecordID. +- `get_job_status()` fetches async job status via surreal-commands library. +- `get_job_detail()` returns both status and error_message from the job (used for retry validation and UI error display). +- `_prepare_save_data()` ensures command field is always RecordID format for database. + +### migration.py + +Data migration for podcast profiles: maps legacy provider/model strings to Model registry record IDs. Runs on API startup after SQL migrations (called from `api/main.py` lifespan). + +- `_find_model_record()`: Finds an existing Model record matching provider + name + type. +- `_find_or_create_model()`: Finds existing Model record or auto-creates one linked to a provider credential. +- `migrate_podcast_profiles()`: Migrates all episode and speaker profiles. Idempotent -- skips profiles where new fields are already populated. Logs counts of migrated/skipped/failed profiles. + +## Common Patterns + +- **Model registry references**: Profile fields reference `record` IDs instead of raw provider/model strings. Credentials are resolved at runtime via `_resolve_model_config()`. +- **Profile snapshots**: episode_profile and speaker_profile stored as dicts on PodcastEpisode to freeze config at generation time. +- **Field validation**: Pydantic validators enforce constraints (segment count, speaker count, required fields). +- **Async database access**: `get_by_name()` queries via repo_query. +- **Job tracking**: command field delegates to surreal-commands; get_job_status() returns "unknown" on failure. +- **Record ID handling**: `_prepare_save_data()` converts model ID strings to RecordID before save; `ensure_record_id()` handles both string and RecordID inputs. +- **nullable_fields ClassVar**: Declares fields that may be null/absent in the database, allowing ObjectModel to handle them during deserialization. + +## Key Dependencies + +- `pydantic`: Field validators, ObjectModel inheritance +- `surrealdb`: RecordID type for job and model references +- `open_notebook.database.repository`: repo_query, ensure_record_id +- `open_notebook.domain.base`: ObjectModel base class +- `open_notebook.ai.models`: Model class (for `_resolve_model_config`) +- `open_notebook.ai.key_provider`: provision_provider_keys (fallback) +- `open_notebook.domain.credential`: Credential (for migration) +- `surreal_commands` (optional): get_command_status() for job status + +## Important Quirks & Gotchas + +- **Legacy fields preserved**: `tts_provider`/`tts_model` on SpeakerProfile and `outline_provider`/`outline_model`/`transcript_provider`/`transcript_model` on EpisodeProfile are kept as optional nullable fields for backward compatibility with the data migration. The app ignores them at runtime. +- **Snapshot approach**: Episode/speaker profiles stored as dicts (not references), so profile updates don't retroactively affect past episodes. +- **Job status resilience**: get_job_status() catches all exceptions and returns "unknown" (no error propagation). +- **No automatic retries**: Podcast generation commands use `retry={"max_attempts": 1}` to prevent duplicate episode records on failure; retry is user-initiated via `POST /podcasts/episodes/{id}/retry`. +- **validate_speakers executes late**: Validators run at instantiation; bulk inserts may not trigger full validation. +- **RecordID coercion**: `_prepare_save_data()` converts model ID strings to RecordID; command field parsed during deserialization. +- **No cascade delete**: Removing a profile doesn't cascade to episodes using it. +- **Migration is idempotent**: `migrate_podcast_profiles()` skips profiles that already have new fields populated. Safe to run multiple times. +- **Migration auto-creates models**: If a legacy provider/model string has no matching Model record but a credential exists for that provider, the migration auto-creates a Model record linked to the credential. + +## How to Extend + +1. **Add new speaker field**: Add to required_fields list in validate_speakers() +2. **Add episode config field**: Validate in EpisodeProfile, update briefing generation code; add to nullable_fields if optional +3. **Add job metadata**: Extend PodcastEpisode with new fields (e.g., progress tracking) +4. **Change job provider**: Replace surreal-commands with alternative job queue library; update get_job_status() +5. **Add new model reference field**: Add field, add to nullable_fields, add RecordID conversion in `_prepare_save_data()`, add resolve method using `_resolve_model_config()` diff --git a/open_notebook/podcasts/__init__.py b/open_notebook/podcasts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..526ebe252304b62412a295a7afb639f769294047 --- /dev/null +++ b/open_notebook/podcasts/__init__.py @@ -0,0 +1,2 @@ +# Podcasts module +# Contains podcast episode models, profiles, and generation logic diff --git a/open_notebook/podcasts/migration.py b/open_notebook/podcasts/migration.py new file mode 100644 index 0000000000000000000000000000000000000000..d238d456cf041688310434a7f5faf1c705749e9f --- /dev/null +++ b/open_notebook/podcasts/migration.py @@ -0,0 +1,189 @@ +""" +Data migration for podcast profiles: maps legacy provider/model strings +to Model registry record IDs. + +Runs on API startup after SQL migrations. Idempotent - skips profiles +that already have the new fields populated. +""" + +from loguru import logger + +from open_notebook.database.repository import repo_query + + +async def _find_model_record( + provider: str, model_name: str, model_type: str +) -> str | None: + """Find an existing Model record matching provider + name + type.""" + results = await repo_query( + "SELECT * FROM model WHERE provider = $provider AND name = $name AND type = $type", + {"provider": provider, "name": model_name, "type": model_type}, + ) + if results: + return str(results[0]["id"]) + return None + + +async def _find_or_create_model( + provider: str, model_name: str, model_type: str +) -> str | None: + """Find existing Model record or auto-create one linked to provider credential.""" + # Try exact match first + model_id = await _find_model_record(provider, model_name, model_type) + if model_id: + return model_id + + # Try to find a credential for this provider and auto-create the model + from open_notebook.domain.credential import Credential + + credentials = await Credential.get_by_provider(provider) + if not credentials: + logger.warning( + f"No credential found for provider '{provider}'. " + f"Cannot auto-create model '{model_name}'. Profile needs manual migration." + ) + return None + + # Use the first credential for the provider + credential = credentials[0] + from open_notebook.ai.models import Model + + model = Model( + name=model_name, + provider=provider, + type=model_type, + credential=str(credential.id), + ) + await model.save() + logger.info( + f"Auto-created model '{model_name}' ({model_type}) " + f"linked to credential '{credential.name}'" + ) + return str(model.id) + + +async def migrate_podcast_profiles() -> None: + """Migrate episode and speaker profiles from legacy strings to Model record IDs. + + Idempotent: skips profiles where new fields are already populated. + """ + logger.info("Starting podcast profile data migration...") + + ep_migrated = 0 + ep_skipped = 0 + ep_failed = 0 + + # Migrate EpisodeProfiles + episode_profiles = await repo_query("SELECT * FROM episode_profile") + for raw in episode_profiles: + profile_name = raw.get("name", raw.get("id", "unknown")) + try: + outline_llm = raw.get("outline_llm") + transcript_llm = raw.get("transcript_llm") + + needs_outline = not outline_llm + needs_transcript = not transcript_llm + + if not needs_outline and not needs_transcript: + ep_skipped += 1 + continue + + updates = {} + + if needs_outline: + outline_provider = raw.get("outline_provider") + outline_model = raw.get("outline_model") + if outline_provider and outline_model: + model_id = await _find_or_create_model( + outline_provider, outline_model, "language" + ) + if model_id: + from open_notebook.database.repository import ensure_record_id + + updates["outline_llm"] = ensure_record_id(model_id) + + if needs_transcript: + transcript_provider = raw.get("transcript_provider") + transcript_model = raw.get("transcript_model") + if transcript_provider and transcript_model: + model_id = await _find_or_create_model( + transcript_provider, transcript_model, "language" + ) + if model_id: + from open_notebook.database.repository import ensure_record_id + + updates["transcript_llm"] = ensure_record_id(model_id) + + if updates: + from open_notebook.database.repository import repo_update + + await repo_update("episode_profile", str(raw["id"]), updates) + ep_migrated += 1 + logger.info( + f"Migrated episode profile '{profile_name}': {list(updates.keys())}" + ) + else: + ep_failed += 1 + logger.warning( + f"Could not migrate episode profile '{profile_name}': " + "no matching models found" + ) + + except Exception as e: + ep_failed += 1 + logger.error(f"Failed to migrate episode profile '{profile_name}': {e}") + + # Migrate SpeakerProfiles + sp_migrated = 0 + sp_skipped = 0 + sp_failed = 0 + + speaker_profiles = await repo_query("SELECT * FROM speaker_profile") + for raw in speaker_profiles: + profile_name = raw.get("name", raw.get("id", "unknown")) + try: + voice_model = raw.get("voice_model") + + if voice_model: + sp_skipped += 1 + continue + + tts_provider = raw.get("tts_provider") + tts_model = raw.get("tts_model") + + if not tts_provider or not tts_model: + sp_failed += 1 + logger.warning( + f"Speaker profile '{profile_name}' has no legacy TTS config" + ) + continue + + model_id = await _find_or_create_model( + tts_provider, tts_model, "text_to_speech" + ) + if model_id: + from open_notebook.database.repository import ensure_record_id, repo_update + + await repo_update( + "speaker_profile", + str(raw["id"]), + {"voice_model": ensure_record_id(model_id)}, + ) + sp_migrated += 1 + logger.info(f"Migrated speaker profile '{profile_name}'") + else: + sp_failed += 1 + logger.warning( + f"Could not migrate speaker profile '{profile_name}': " + "no matching model found" + ) + + except Exception as e: + sp_failed += 1 + logger.error(f"Failed to migrate speaker profile '{profile_name}': {e}") + + logger.info( + f"Podcast profile migration complete. " + f"Episodes: {ep_migrated} migrated, {ep_skipped} skipped, {ep_failed} failed. " + f"Speakers: {sp_migrated} migrated, {sp_skipped} skipped, {sp_failed} failed." + ) diff --git a/open_notebook/podcasts/models.py b/open_notebook/podcasts/models.py new file mode 100644 index 0000000000000000000000000000000000000000..cccdf9e31ea8ec9054cd2797175bbda55661832e --- /dev/null +++ b/open_notebook/podcasts/models.py @@ -0,0 +1,277 @@ +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Union + +from loguru import logger +from pydantic import ConfigDict, Field, field_validator +from surrealdb import RecordID + +from open_notebook.database.repository import ensure_record_id, repo_query +from open_notebook.domain.base import ObjectModel + + +async def _resolve_model_config(model_id: str) -> Tuple[str, str, dict]: + """Load Model record, resolve credential -> (provider, model_name, config_dict). + + Used by resolve_outline_config, resolve_transcript_config, resolve_tts_config, + and per-speaker TTS overrides. + """ + from open_notebook.ai.models import Model + + model = await Model.get(model_id) + config: dict = {} + if model.credential: + credential = await model.get_credential_obj() + if credential: + config = credential.to_esperanto_config() + if not config: + from open_notebook.ai.key_provider import provision_provider_keys + + await provision_provider_keys(model.provider) + return (model.provider, model.name, config) + + +class EpisodeProfile(ObjectModel): + """ + Episode Profile - Simplified podcast configuration. + Replaces complex 15+ field configuration with user-friendly profiles. + """ + + table_name: ClassVar[str] = "episode_profile" + nullable_fields: ClassVar[set[str]] = { + "description", + "outline_provider", + "outline_model", + "transcript_provider", + "transcript_model", + "outline_llm", + "transcript_llm", + "language", + } + + name: str = Field(..., description="Unique profile name") + description: Optional[str] = Field(None, description="Profile description") + speaker_config: str = Field(..., description="Reference to speaker profile name") + + # Legacy fields (kept for migration, app ignores) + outline_provider: Optional[str] = Field( + None, description="[Legacy] AI provider for outline generation" + ) + outline_model: Optional[str] = Field( + None, description="[Legacy] AI model for outline generation" + ) + transcript_provider: Optional[str] = Field( + None, description="[Legacy] AI provider for transcript generation" + ) + transcript_model: Optional[str] = Field( + None, description="[Legacy] AI model for transcript generation" + ) + + # New fields: Model registry references + outline_llm: Optional[str] = Field( + None, description="Model record ID for outline generation" + ) + transcript_llm: Optional[str] = Field( + None, description="Model record ID for transcript generation" + ) + language: Optional[str] = Field( + None, description="Podcast language (BCP 47 locale code, e.g. pt-BR, en-US)" + ) + + default_briefing: str = Field(..., description="Default briefing template") + num_segments: int = Field(default=5, description="Number of podcast segments") + + @field_validator("num_segments") + @classmethod + def validate_segments(cls, v): + if not 3 <= v <= 20: + raise ValueError("Number of segments must be between 3 and 20") + return v + + def _prepare_save_data(self) -> dict: + data = super()._prepare_save_data() + if data.get("outline_llm"): + data["outline_llm"] = ensure_record_id(data["outline_llm"]) + if data.get("transcript_llm"): + data["transcript_llm"] = ensure_record_id(data["transcript_llm"]) + return data + + async def resolve_outline_config(self) -> Tuple[str, str, dict]: + """Resolve outline model -> (provider, model_name, config_dict)""" + if not self.outline_llm: + raise ValueError( + f"Episode profile '{self.name}' has no outline model configured. " + "Please update the profile to select an outline model." + ) + return await _resolve_model_config(self.outline_llm) + + async def resolve_transcript_config(self) -> Tuple[str, str, dict]: + """Resolve transcript model -> (provider, model_name, config_dict)""" + if not self.transcript_llm: + raise ValueError( + f"Episode profile '{self.name}' has no transcript model configured. " + "Please update the profile to select a transcript model." + ) + return await _resolve_model_config(self.transcript_llm) + + @classmethod + async def get_by_name(cls, name: str) -> Optional["EpisodeProfile"]: + """Get episode profile by name""" + result = await repo_query( + "SELECT * FROM episode_profile WHERE name = $name", {"name": name} + ) + if result: + return cls(**result[0]) + return None + + +class SpeakerProfile(ObjectModel): + """ + Speaker Profile - Voice and personality configuration. + Supports 1-4 speakers for flexible podcast formats. + """ + + table_name: ClassVar[str] = "speaker_profile" + nullable_fields: ClassVar[set[str]] = { + "description", + "tts_provider", + "tts_model", + "voice_model", + } + + name: str = Field(..., description="Unique profile name") + description: Optional[str] = Field(None, description="Profile description") + + # Legacy fields (kept for migration, app ignores) + tts_provider: Optional[str] = Field( + None, description="[Legacy] TTS provider (openai, elevenlabs, etc.)" + ) + tts_model: Optional[str] = Field(None, description="[Legacy] TTS model name") + + # New field: Model registry reference + voice_model: Optional[str] = Field( + None, description="Model record ID for TTS" + ) + + speakers: List[Dict[str, Any]] = Field( + ..., description="Array of speaker configurations" + ) + + @field_validator("speakers") + @classmethod + def validate_speakers(cls, v): + if not 1 <= len(v) <= 4: + raise ValueError("Must have between 1 and 4 speakers") + + required_fields = ["name", "voice_id", "backstory", "personality"] + for speaker in v: + for field in required_fields: + if field not in speaker: + raise ValueError(f"Speaker missing required field: {field}") + return v + + def _prepare_save_data(self) -> dict: + data = super()._prepare_save_data() + if data.get("voice_model"): + data["voice_model"] = ensure_record_id(data["voice_model"]) + # Handle per-speaker voice_model overrides + if data.get("speakers"): + for speaker in data["speakers"]: + if speaker.get("voice_model"): + speaker["voice_model"] = ensure_record_id(speaker["voice_model"]) + return data + + async def resolve_tts_config(self) -> Tuple[str, str, dict]: + """Resolve TTS model -> (provider, model_name, config_dict)""" + if not self.voice_model: + raise ValueError( + f"Speaker profile '{self.name}' has no voice model configured. " + "Please update the profile to select a voice model." + ) + return await _resolve_model_config(self.voice_model) + + @classmethod + async def get_by_name(cls, name: str) -> Optional["SpeakerProfile"]: + """Get speaker profile by name""" + result = await repo_query( + "SELECT * FROM speaker_profile WHERE name = $name", {"name": name} + ) + if result: + return cls(**result[0]) + return None + + +class PodcastEpisode(ObjectModel): + """Enhanced PodcastEpisode with job tracking and metadata""" + + table_name: ClassVar[str] = "episode" + + name: str = Field(..., description="Episode name") + episode_profile: Dict[str, Any] = Field( + ..., description="Episode profile used (stored as object)" + ) + speaker_profile: Dict[str, Any] = Field( + ..., description="Speaker profile used (stored as object)" + ) + briefing: str = Field(..., description="Full briefing used for generation") + content: str = Field(..., description="Source content") + audio_file: Optional[str] = Field( + default=None, description="Path to generated audio file" + ) + transcript: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="Generated transcript" + ) + outline: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="Generated outline" + ) + command: Optional[Union[str, RecordID]] = Field( + default=None, description="Link to surreal-commands job" + ) + + model_config = ConfigDict(arbitrary_types_allowed=True) + + async def get_job_status(self) -> Optional[str]: + """Get the status of the associated command""" + if not self.command: + return None + + try: + from surreal_commands import get_command_status + + status = await get_command_status(str(self.command)) + return status.status if status else "unknown" + except Exception: + return "unknown" + + async def get_job_detail(self) -> dict: + """Get status and error_message of the associated command""" + if not self.command: + return {"status": None, "error_message": None} + + try: + from surreal_commands import get_command_status + + status = await get_command_status(str(self.command)) + if not status: + return {"status": "unknown", "error_message": None} + return { + "status": status.status, + "error_message": getattr(status, "error_message", None), + } + except Exception: + return {"status": "unknown", "error_message": None} + + @field_validator("command", mode="before") + @classmethod + def parse_command(cls, value): + if isinstance(value, str): + return ensure_record_id(value) + return value + + def _prepare_save_data(self) -> dict: + """Override to ensure command field is always RecordID format for database""" + data = super()._prepare_save_data() + + # Ensure command field is RecordID format if not None + if data.get("command") is not None: + data["command"] = ensure_record_id(data["command"]) + + return data diff --git a/open_notebook/utils/CLAUDE.md b/open_notebook/utils/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..ec4d08326728d0475f3692c456fac605e0e7ce27 --- /dev/null +++ b/open_notebook/utils/CLAUDE.md @@ -0,0 +1,229 @@ +# Utils Module + +Utility functions and helpers for context building, text processing, chunking, embedding, tokenization, and versioning. + +## Purpose + +Provides cross-cutting concerns: building LLM context from sources/insights, content-type aware text chunking, unified embedding generation with mean pooling, token counting, and version management. + +## Architecture Overview + +**Six core utilities**: +1. **context_builder.py**: Flexible context assembly from sources, notes, insights with token budgeting +2. **chunking.py**: Content-type detection and smart text chunking for embedding operations +3. **embedding.py**: Unified embedding generation with mean pooling for large content +4. **text_utils.py**: Text cleaning and thinking content extraction +5. **token_utils.py**: Token counting for LLM context windows (wrapper around encoding library) +6. **version_utils.py**: Version parsing, comparison, and schema compatibility checks + +Each utility is stateless and can be imported independently. + +## Configuration + +### Chunking Configuration (chunking.py) + +The chunking behavior can be configured via environment variables: + +- **OPEN_NOTEBOOK_CHUNK_SIZE**: Maximum chunk size in tokens (default: 400) + - Minimum: 100 tokens + - Warnings: Values > 8192 tokens or invalid values + - Use case: Conservative baseline that leaves headroom below 512-token embedders (e.g. mxbai-embed-large). Buffer accounts for tokenizer mismatch between our `o200k_base` measurement and the embedder's own tokenizer, plus occasional splitter overshoot and special tokens. + +- **OPEN_NOTEBOOK_CHUNK_OVERLAP**: Overlap between chunks in tokens (default: 15% of CHUNK_SIZE) + - Must be: >= 0 and < CHUNK_SIZE + - Warnings: Invalid values or values >= CHUNK_SIZE + - Use case: Control how much context is shared between adjacent chunks + +Example for embedders with larger context windows (e.g. OpenAI text-embedding-3 family, 8191 tokens): +```bash +export OPEN_NOTEBOOK_CHUNK_SIZE=1500 +export OPEN_NOTEBOOK_CHUNK_OVERLAP=150 +``` + +Note: Changes require restart of the application. + +## Component Catalog + +### context_builder.py +- **ContextItem**: Dataclass for individual context piece (id, type, content, priority, token_count) +- **ContextConfig**: Configuration for context building (sources/notes/insights selection, max tokens, priority weights) +- **ContextBuilder**: Main class assembling context + - `add_source()`: Include source by ID with inclusion level + - `add_note()`: Include note by ID + - `add_insight()`: Include insight by ID + - `build()`: Assemble context respecting token budget and priorities + - Uses vector_search to fetch source/insight content from SurrealDB + - Returns list of ContextItem objects sorted by priority + +**Key behavior**: +- Token counting is automatic (calculated in ContextItem.__post_init__) +- Max token enforcement via priority weighting (higher priority items included first) +- Type-specific fetching: sources → Source.full_text, notes → Note.content, insights → SourceInsight.content +- Raises DatabaseOperationError if source/note fetch fails + +### chunking.py +- **ContentType**: Enum (HTML, MARKDOWN, PLAIN) +- **CHUNK_SIZE**: Configurable via `OPEN_NOTEBOOK_CHUNK_SIZE` env var (default: 400) +- **CHUNK_OVERLAP**: Configurable via `OPEN_NOTEBOOK_CHUNK_OVERLAP` env var (default: 15% of CHUNK_SIZE) +- **detect_content_type_from_extension(file_path)**: Detect type from file extension +- **detect_content_type_from_heuristics(text)**: Detect type from content patterns (returns type + confidence) +- **detect_content_type(text, file_path)**: Combined detection (extension primary, heuristics fallback) +- **chunk_text(text, content_type, file_path)**: Split text using appropriate splitter + +**Key behavior**: +- Uses LangChain splitters: HTMLHeaderTextSplitter, MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter +- Extension-based detection is primary; heuristics can override PLAIN extensions with 0.8+ confidence +- Secondary chunking applied when HTML/Markdown splitters produce oversized chunks +- Returns list of strings, each approximately ≤ CHUNK_SIZE tokens + +### embedding.py +- **mean_pool_embeddings(embeddings)**: Combine multiple embeddings via normalized mean pooling +- **generate_embeddings(texts)**: Batch embedding with automatic batching (default 50 texts per batch) and per-batch retry +- **generate_embedding(text, content_type, file_path)**: Unified embedding with automatic chunking + mean pooling + +**Key behavior**: +- Uses model_manager.get_model("embedding") for embedding model +- Short text (≤ CHUNK_SIZE tokens): direct embedding +- Long text: chunk → embed each → mean pool results +- Mean pooling: normalize each → mean → normalize result (using numpy) +- Raises ValueError for empty/whitespace-only text + +### text_utils.py +- **remove_non_ascii(text)**: Remove non-ASCII characters from text +- **remove_non_printable(text)**: Remove non-printable characters, preserving newlines/tabs +- **parse_thinking_content(content)**: Extract `` tags content from AI responses +- **clean_thinking_content(content)**: Remove `` blocks, return cleaned content only + +**Key behavior**: +- parse_thinking_content handles malformed output (missing opening `` tag) +- Large content (>100KB) bypasses thinking extraction for performance +- Non-string input returns empty thinking and stringified content + +### token_utils.py +- **token_count(text)**: Returns estimated token count for string (via tiktoken) +- **token_cost(text, model)**: Calculate cost estimate for text with given model + +**Key behavior**: Uses `o200k_base` encoding; may differ slightly from actual model tokenization. If `tiktoken` is unavailable, `token_count()` falls back to a coarse estimate; this refactor keeps that existing contract. + +### version_utils.py +- **compare_versions(v1, v2)**: Returns -1 (v1 < v2), 0 (equal), 1 (v1 > v2) +- **get_installed_version(package)**: Get version of installed Python package +- **get_version_from_github(url)**: Fetch latest version from GitHub releases + +**Key behavior**: Uses packaging library for version parsing; supports pre-release tags + +## Common Patterns + +- **Dataclass-driven config**: ContextConfig used by ContextBuilder (immutable after init) +- **Token budgeting**: ContextBuilder respects max_tokens constraint; prioritizes high-priority items +- **Content-type aware processing**: Chunking uses appropriate splitter based on detected content type +- **Mean pooling for large content**: Embedding handles arbitrarily large text via chunking + pooling +- **Error handling resilience**: token_count() returns estimate; context_builder catches DB errors gracefully +- **Pure text functions**: text_utils functions are stateless utilities (no class needed) +- **Lazy evaluation**: ContextBuilder doesn't fetch items until build() called +- **Type hints throughout**: All functions use Optional, List, Dict for clarity + +## Key Dependencies + +- `open_notebook.domain.notebook`: Source, Note, SourceInsight models; vector_search function +- `open_notebook.ai.models`: model_manager for embedding model access +- `open_notebook.exceptions`: DatabaseOperationError, NotFoundError +- `langchain_text_splitters`: HTMLHeaderTextSplitter, MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter +- `numpy`: Mean pooling calculations +- `tiktoken`: Token encoding for GPT models +- `loguru`: Logging throughout + +## Important Quirks & Gotchas + +- **Token count estimation**: Uses `o200k_base` encoding; may differ slightly from actual model tokens +- **Chunk size semantics changed**: `OPEN_NOTEBOOK_CHUNK_SIZE` and `OPEN_NOTEBOOK_CHUNK_OVERLAP` are token-based, not character-based +- **Default chunk size**: The token-based default is 400 — leaves ~20% margin below the 512-token ceiling of BERT-family embedders (e.g. mxbai-embed-large) to absorb tokenizer mismatch (we measure with `o200k_base`, they tokenize with WordPiece), splitter overshoot, and special tokens +- **Content type detection order**: Extension checked first, then heuristics; high-confidence heuristics (≥0.8) can override PLAIN extensions +- **Mean pooling normalization**: Each embedding normalized before mean, result normalized after +- **Priority weights default**: If not specified, ContextConfig uses default weights (source=1, note=0.8, insight=1.2) +- **Vector search required**: ContextBuilder assumes vector_search is available on Notebook model; fails if not +- **Circular import risk**: context_builder imports from domain.notebook; avoid domain importing utils +- **Max tokens hard limit**: ContextBuilder stops adding items once max_tokens exceeded (not prorated) +- **No caching**: Every build() call re-fetches from database (use cache layer if needed) + +## How to Extend + +1. **Add new context source type**: Create fetch method in ContextBuilder; update ContextConfig.sources dict +2. **Add content type**: Add to ContentType enum; create splitter getter; update chunk_text() +3. **Change chunk size**: Set OPEN_NOTEBOOK_CHUNK_SIZE and OPEN_NOTEBOOK_CHUNK_OVERLAP environment variables +4. **Add text preprocessing**: Add new function to text_utils (e.g., remove_urls, extract_keywords) +5. **Change tokenization**: Replace tiktoken with alternative library in token_utils; update all calls +6. **Add context filtering**: Extend ContextConfig with filter_by_date, filter_by_topic fields + +## Usage Examples + +### Chunking +```python +from open_notebook.utils.chunking import chunk_text, detect_content_type, ContentType + +# Auto-detect content type and chunk +chunks = chunk_text(long_text, file_path="document.md") + +# Explicit content type +chunks = chunk_text(html_content, content_type=ContentType.HTML) +``` + +### Embedding +```python +from open_notebook.utils.embedding import generate_embedding, generate_embeddings + +# Single text (handles chunking + mean pooling automatically) +embedding = await generate_embedding(long_text) + +# Batch embedding (more efficient for multiple texts) +embeddings = await generate_embeddings(["text1", "text2", "text3"]) +``` + +### Context Building +```python +from open_notebook.utils.context_builder import ContextBuilder, ContextConfig + +config = ContextConfig( + sources={"source:123": "full", "source:456": "summary"}, + max_tokens=2000, +) +builder = ContextBuilder(notebook, config) +context_items = await builder.build() + +for item in context_items: + print(f"{item.type}:{item.id} ({item.token_count} tokens)") +``` + +### encryption.py +- **get_secret_from_env(var_name)**: Retrieve secret from environment with Docker secrets support (checks VAR_FILE first, then VAR) +- **get_fernet()**: Get Fernet instance if encryption key is configured +- **encrypt_value(value)**: Encrypt a string using Fernet symmetric encryption +- **decrypt_value(value)**: Decrypt a Fernet-encrypted string; gracefully falls back to original value for legacy/unencrypted data +**Purpose**: Provides field-level encryption for sensitive data (API keys) stored in the database. Uses Fernet symmetric encryption (AES-128-CBC with HMAC-SHA256) for authenticated encryption. + +**Key behavior**: +- Key source: OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE (Docker secrets) → OPEN_NOTEBOOK_ENCRYPTION_KEY (env var) +- Accepts **any string**: always derived to a Fernet key via SHA-256 +- No default key — encryption is unavailable until the env var is set +- Graceful fallback on decryption: InvalidToken errors (legacy unencrypted data) return the original value +- Lazy-loaded key: initialized on first use, not at import time + +**Security considerations**: +- OPEN_NOTEBOOK_ENCRYPTION_KEY must be set explicitly (no default) +- Docker secrets pattern supported for secure key injection in containerized environments +- Key rotation would require re-encrypting all stored keys (not currently implemented) +- Encryption is transparent to callers; unencrypted legacy data continues to work + +**Usage Example**: +```python +from open_notebook.utils.encryption import encrypt_value, decrypt_value + +# Encrypt before storing in database +encrypted_api_key = encrypt_value(api_key) + +# Decrypt when reading from database +decrypted_api_key = decrypt_value(encrypted_api_key) + +# Set any string as encryption key: +# OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret-passphrase +``` diff --git a/open_notebook/utils/README.md b/open_notebook/utils/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0316b9f5b7177685f21e983f125146d89a2936c4 --- /dev/null +++ b/open_notebook/utils/README.md @@ -0,0 +1,188 @@ +# ContextBuilder + +A flexible and generic ContextBuilder class for the Open Notebook project that can handle any parameters and build context from sources, notebooks, insights, and notes. + +## Features + +- **Flexible Parameters**: Accepts any parameters via `**kwargs` for future extensibility +- **Priority-based Management**: Automatic prioritization and sorting of context items +- **Token Counting**: Built-in token counting and truncation to fit limits +- **Deduplication**: Automatic removal of duplicate items based on ID +- **Type-based Grouping**: Separates sources, notes, and insights in output +- **Async Support**: Fully async for database operations + +## Basic Usage + +```python +from open_notebook.utils.context_builder import ContextBuilder, ContextConfig + +# Simple notebook context +builder = ContextBuilder(notebook_id="notebook:123") +context = await builder.build() + +# Single source with insights +builder = ContextBuilder( + source_id="source:456", + include_insights=True, + max_tokens=2000 +) +context = await builder.build() +``` + +## Convenience Functions + +```python +from open_notebook.utils.context_builder import ( + build_notebook_context, + build_source_context, + build_mixed_context +) + +# Build notebook context +context = await build_notebook_context( + notebook_id="notebook:123", + max_tokens=5000 +) + +# Build single source context +context = await build_source_context( + source_id="source:456", + include_insights=True +) + +# Build mixed context +context = await build_mixed_context( + source_ids=["source:1", "source:2"], + note_ids=["note:1", "note:2"], + max_tokens=3000 +) +``` + +## Advanced Configuration + +```python +from open_notebook.utils.context_builder import ContextConfig + +# Custom configuration +config = ContextConfig( + sources={ + "source:doc1": "insights", + "source:doc2": "full content", + "source:doc3": "not in" # Exclude + }, + notes={ + "note:summary": "full content", + "note:draft": "not in" # Exclude + }, + include_insights=True, + max_tokens=3000, + priority_weights={ + "source": 120, # Higher priority + "note": 80, # Medium priority + "insight": 100 # High priority + } +) + +builder = ContextBuilder( + notebook_id="notebook:project", + context_config=config +) +context = await builder.build() +``` + +## Programmatic Item Management + +```python +from open_notebook.utils.context_builder import ContextItem + +builder = ContextBuilder() + +# Add custom items +item = ContextItem( + id="source:important", + type="source", + content={"title": "Key Document", "summary": "..."}, + priority=150 # Very high priority +) +builder.add_item(item) + +# Apply management operations +builder.remove_duplicates() +builder.prioritize() +builder.truncate_to_fit(1000) + +context = builder._format_response() +``` + +## Flexible Parameters + +The ContextBuilder accepts any parameters via `**kwargs`, making it extensible for future features: + +```python +builder = ContextBuilder( + notebook_id="notebook:123", + include_insights=True, + max_tokens=2000, + + # Custom parameters for future extensions + user_id="user:456", + custom_filter="advanced", + experimental_feature=True +) + +# Access custom parameters +user_id = builder.params.get('user_id') +``` + +## Output Format + +The ContextBuilder returns a structured response: + +```python +{ + "sources": [...], # List of source contexts + "notes": [...], # List of note contexts + "insights": [...], # List of insight contexts + "total_tokens": 1234, # Total token count + "total_items": 10, # Total number of items + "notebook_id": "notebook:123", # If provided + "metadata": { + "source_count": 5, + "note_count": 3, + "insight_count": 2, + "config": { + "include_insights": true, + "include_notes": true, + "max_tokens": 2000 + } + } +} +``` + +## Architecture + +The ContextBuilder follows these design principles: + +1. **Separation of Concerns**: Context building, item management, and formatting are separate +2. **Extensibility**: Uses `**kwargs` and flexible configuration for future features +3. **Performance**: Token-aware truncation and efficient deduplication +4. **Type Safety**: Proper type hints and data classes for structure +5. **Error Handling**: Graceful handling of missing items and database errors + +## Integration + +The ContextBuilder integrates seamlessly with the existing Open Notebook architecture: + +- Uses existing domain models (`Source`, `Notebook`, `Note`) +- Leverages the repository pattern for database access +- Follows the same async patterns as other services +- Integrates with the token counting utilities + +## Error Handling + +The ContextBuilder handles errors gracefully: + +- Missing notebooks/sources/notes are logged but don't stop execution +- Database errors are wrapped in `DatabaseOperationError` +- Invalid parameters raise `InvalidInputError` +- All errors include detailed context information \ No newline at end of file diff --git a/open_notebook/utils/__init__.py b/open_notebook/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a31c1bda84834cf6160c291b5c9555fe3dc1a234 --- /dev/null +++ b/open_notebook/utils/__init__.py @@ -0,0 +1,69 @@ +""" +Utils package for Open Notebook. + +To avoid circular imports, import functions directly: +- from open_notebook.utils.context_builder import ContextBuilder +- from open_notebook.utils import token_count, compare_versions +- from open_notebook.utils.chunking import chunk_text, detect_content_type, ContentType +- from open_notebook.utils.embedding import generate_embedding, generate_embeddings +- from open_notebook.utils.encryption import encrypt_value, decrypt_value +""" + +from .chunking import ( + CHUNK_SIZE, + ContentType, + chunk_text, + detect_content_type, + detect_content_type_from_extension, + detect_content_type_from_heuristics, +) +from .embedding import ( + generate_embedding, + generate_embeddings, + mean_pool_embeddings, +) +from .encryption import ( + decrypt_value, + encrypt_value, +) +from .text_utils import ( + clean_thinking_content, + parse_thinking_content, + remove_non_ascii, + remove_non_printable, +) +from .token_utils import token_cost, token_count +from .version_utils import ( + compare_versions, + get_installed_version, + get_version_from_github, +) + +__all__ = [ + # Chunking + "CHUNK_SIZE", + "ContentType", + "chunk_text", + "detect_content_type", + "detect_content_type_from_extension", + "detect_content_type_from_heuristics", + # Embedding + "generate_embedding", + "generate_embeddings", + "mean_pool_embeddings", + # Text utils + "remove_non_ascii", + "remove_non_printable", + "parse_thinking_content", + "clean_thinking_content", + # Token utils + "token_count", + "token_cost", + # Version utils + "compare_versions", + "get_installed_version", + "get_version_from_github", + # Encryption utils + "decrypt_value", + "encrypt_value", +] diff --git a/open_notebook/utils/chunking.py b/open_notebook/utils/chunking.py new file mode 100644 index 0000000000000000000000000000000000000000..b0eb6fe252f37ca17edcd6b9bd73427c1a46df05 --- /dev/null +++ b/open_notebook/utils/chunking.py @@ -0,0 +1,448 @@ +""" +Chunking utilities for Open Notebook. + +Provides content-type detection and smart text chunking for embedding operations. +Supports HTML, Markdown, and plain text with appropriate splitters for each type. + +Key functions: +- detect_content_type(): Detects content type from file extension or content heuristics +- chunk_text(): Splits text into chunks using appropriate splitter for content type + +Environment Variables: + OPEN_NOTEBOOK_CHUNK_SIZE: Maximum chunk size in tokens (default: 400) + OPEN_NOTEBOOK_CHUNK_OVERLAP: Overlap between chunks in tokens (default: 15% of CHUNK_SIZE) +""" + +import os +import re +from enum import Enum +from pathlib import Path +from typing import List, Optional, Tuple + +from langchain_text_splitters import ( + HTMLHeaderTextSplitter, + MarkdownHeaderTextSplitter, + RecursiveCharacterTextSplitter, +) +from loguru import logger + +from .token_utils import token_count + + +def _get_chunk_size() -> int: + """Get chunk size from environment variable or use default.""" + chunk_size_str = os.getenv("OPEN_NOTEBOOK_CHUNK_SIZE") + if chunk_size_str: + try: + chunk_size = int(chunk_size_str) + if chunk_size < 100: + logger.warning( + f"OPEN_NOTEBOOK_CHUNK_SIZE ({chunk_size}) is too small. " + f"Using minimum value of 100." + ) + return 100 + if chunk_size > 8192: + logger.warning( + f"OPEN_NOTEBOOK_CHUNK_SIZE ({chunk_size}) is very large. " + f"This may cause issues with some embedding models." + ) + logger.info(f"Using custom chunk size: {chunk_size} tokens") + return chunk_size + except ValueError: + logger.warning( + f"Invalid OPEN_NOTEBOOK_CHUNK_SIZE value: '{chunk_size_str}'. " + f"Using default: 400" + ) + return 400 + + +def _get_chunk_overlap(chunk_size: int) -> int: + """Get chunk overlap from environment variable or calculate default (15% of chunk size).""" + overlap_str = os.getenv("OPEN_NOTEBOOK_CHUNK_OVERLAP") + if overlap_str: + try: + overlap = int(overlap_str) + if overlap < 0: + logger.warning( + f"OPEN_NOTEBOOK_CHUNK_OVERLAP ({overlap}) cannot be negative. " + f"Using 0." + ) + return 0 + if overlap >= chunk_size: + logger.warning( + f"OPEN_NOTEBOOK_CHUNK_OVERLAP ({overlap}) cannot be >= chunk size ({chunk_size}). " + f"Using 15% of chunk size: {int(chunk_size * 0.15)}" + ) + return int(chunk_size * 0.15) + logger.info(f"Using custom chunk overlap: {overlap} tokens") + return overlap + except ValueError: + logger.warning( + f"Invalid OPEN_NOTEBOOK_CHUNK_OVERLAP value: '{overlap_str}'. " + f"Using default: 15% of chunk size" + ) + return int(chunk_size * 0.15) + + +# Constants (computed at import time from environment variables) +CHUNK_SIZE = _get_chunk_size() +CHUNK_OVERLAP = _get_chunk_overlap(CHUNK_SIZE) +HIGH_CONFIDENCE_THRESHOLD = 0.8 # Threshold for heuristics to override extension + +logger.debug( + f"Chunking configuration: CHUNK_SIZE={CHUNK_SIZE}, CHUNK_OVERLAP={CHUNK_OVERLAP}" +) + + +class ContentType(Enum): + """Content type for chunking strategy selection.""" + + HTML = "html" + MARKDOWN = "markdown" + PLAIN = "plain" + + +# File extension mappings +_EXTENSION_TO_CONTENT_TYPE = { + # HTML + ".html": ContentType.HTML, + ".htm": ContentType.HTML, + ".xhtml": ContentType.HTML, + # Markdown + ".md": ContentType.MARKDOWN, + ".markdown": ContentType.MARKDOWN, + ".mdown": ContentType.MARKDOWN, + ".mkd": ContentType.MARKDOWN, + # Plain text (explicit) + ".txt": ContentType.PLAIN, + ".text": ContentType.PLAIN, + # Code files (treat as plain) + ".py": ContentType.PLAIN, + ".js": ContentType.PLAIN, + ".ts": ContentType.PLAIN, + ".java": ContentType.PLAIN, + ".c": ContentType.PLAIN, + ".cpp": ContentType.PLAIN, + ".go": ContentType.PLAIN, + ".rs": ContentType.PLAIN, + ".rb": ContentType.PLAIN, + ".php": ContentType.PLAIN, + ".sh": ContentType.PLAIN, + ".bash": ContentType.PLAIN, + ".zsh": ContentType.PLAIN, + ".sql": ContentType.PLAIN, + ".json": ContentType.PLAIN, + ".yaml": ContentType.PLAIN, + ".yml": ContentType.PLAIN, + ".xml": ContentType.PLAIN, + ".csv": ContentType.PLAIN, + ".tsv": ContentType.PLAIN, +} + + +def detect_content_type_from_extension( + file_path: Optional[str], +) -> Optional[ContentType]: + """ + Detect content type from file extension. + + Args: + file_path: Path to the file (can be full path or just filename) + + Returns: + ContentType if extension is recognized, None otherwise + """ + if not file_path: + return None + + try: + extension = Path(file_path).suffix.lower() + return _EXTENSION_TO_CONTENT_TYPE.get(extension) + except Exception: + return None + + +def detect_content_type_from_heuristics(text: str) -> Tuple[ContentType, float]: + """ + Detect content type using content heuristics. + + Args: + text: The text content to analyze + + Returns: + Tuple of (ContentType, confidence_score) where confidence is 0.0-1.0 + """ + if not text or len(text) < 10: + return ContentType.PLAIN, 0.5 + + # Sample first 5000 chars for efficiency + sample = text[:5000] + + # Check HTML first (most specific patterns) + html_score = _calculate_html_score(sample) + if html_score >= HIGH_CONFIDENCE_THRESHOLD: + return ContentType.HTML, html_score + + # Check Markdown + markdown_score = _calculate_markdown_score(sample) + if markdown_score >= HIGH_CONFIDENCE_THRESHOLD: + return ContentType.MARKDOWN, markdown_score + + # Return the higher scoring type, or PLAIN if both are low + if html_score > markdown_score and html_score > 0.3: + return ContentType.HTML, html_score + elif markdown_score > 0.3: + return ContentType.MARKDOWN, markdown_score + else: + return ContentType.PLAIN, 0.6 + + +def _calculate_html_score(text: str) -> float: + """Calculate confidence score for HTML content.""" + score = 0.0 + indicators = 0 + + # Strong indicators + if re.search(r"]", text, re.IGNORECASE): + score += 0.3 + indicators += 1 + + # Structural tags + structural_tags = ["", "= 5: + break + + # Header tags + if re.search(r"]", text, re.IGNORECASE): + score += 0.15 + indicators += 1 + + # Closing tags pattern + if re.search(r"", text): + score += 0.1 + indicators += 1 + + return min(score, 1.0) + + +def _calculate_markdown_score(text: str) -> float: + """Calculate confidence score for Markdown content.""" + score = 0.0 + indicators = 0 + + # Headers (# ## ###) - strong indicator + header_matches = len(re.findall(r"^#{1,6}\s+.+", text, re.MULTILINE)) + if header_matches >= 3: + score += 0.35 + indicators += 1 + elif header_matches >= 1: + score += 0.2 + indicators += 1 + + # Links [text](url) - strong indicator + link_matches = len(re.findall(r"\[.+?\]\(.+?\)", text)) + if link_matches >= 2: + score += 0.25 + indicators += 1 + elif link_matches >= 1: + score += 0.15 + indicators += 1 + + # Code blocks ``` - strong indicator + if re.search(r"^```", text, re.MULTILINE): + score += 0.2 + indicators += 1 + + # Inline code `code` + if re.search(r"`[^`]+`", text): + score += 0.1 + indicators += 1 + + # Lists (-, *, +, or numbered) + list_matches = len(re.findall(r"^[\*\-\+]\s+", text, re.MULTILINE)) + list_matches += len(re.findall(r"^\d+\.\s+", text, re.MULTILINE)) + if list_matches >= 3: + score += 0.15 + indicators += 1 + elif list_matches >= 1: + score += 0.08 + indicators += 1 + + # Bold/italic + if re.search(r"\*\*.+?\*\*|__.+?__", text): + score += 0.1 + indicators += 1 + + # Blockquotes + if re.search(r"^>\s+", text, re.MULTILINE): + score += 0.1 + indicators += 1 + + return min(score, 1.0) + + +def detect_content_type(text: str, file_path: Optional[str] = None) -> ContentType: + """ + Detect content type using file extension (primary) and heuristics (fallback). + + Strategy: + 1. If file extension is available and recognized, use it as primary + 2. If no extension or generic extension (.txt), use heuristics + 3. Heuristics can override extension only with very high confidence + + Args: + text: The text content + file_path: Optional file path for extension-based detection + + Returns: + Detected ContentType + """ + # Try extension-based detection first + extension_type = detect_content_type_from_extension(file_path) + + # Get heuristic-based detection + heuristic_type, confidence = detect_content_type_from_heuristics(text) + + # If no extension or generic extension, use heuristics + if extension_type is None: + logger.debug( + f"No file extension, using heuristics: {heuristic_type.value} " + f"(confidence: {confidence:.2f})" + ) + return heuristic_type + + # If extension suggests plain text but heuristics are very confident, override + if extension_type == ContentType.PLAIN and confidence >= HIGH_CONFIDENCE_THRESHOLD: + logger.debug( + f"Extension suggests plain, but heuristics override with " + f"{heuristic_type.value} (confidence: {confidence:.2f})" + ) + return heuristic_type + + # Otherwise trust the extension + logger.debug(f"Using extension-based content type: {extension_type.value}") + return extension_type + + +def _get_html_splitter() -> HTMLHeaderTextSplitter: + """Get HTML header splitter configured for h1, h2, h3.""" + headers_to_split_on = [ + ("h1", "Header 1"), + ("h2", "Header 2"), + ("h3", "Header 3"), + ] + return HTMLHeaderTextSplitter(headers_to_split_on=headers_to_split_on) + + +def _get_markdown_splitter() -> MarkdownHeaderTextSplitter: + """Get Markdown header splitter configured for #, ##, ###.""" + headers_to_split_on = [ + ("#", "Header 1"), + ("##", "Header 2"), + ("###", "Header 3"), + ] + return MarkdownHeaderTextSplitter( + headers_to_split_on=headers_to_split_on, + strip_headers=False, + ) + + +def _get_plain_splitter() -> RecursiveCharacterTextSplitter: + """Get plain text splitter using CHUNK_SIZE and CHUNK_OVERLAP constants.""" + return RecursiveCharacterTextSplitter( + chunk_size=CHUNK_SIZE, + chunk_overlap=CHUNK_OVERLAP, + length_function=token_count, + separators=["\n\n", "\n", ". ", ", ", " ", ""], + ) + + +def _apply_secondary_chunking(chunks: List[str]) -> List[str]: + """ + Apply secondary chunking to ensure no chunk exceeds CHUNK_SIZE tokens. + + Used when primary splitters (HTML/Markdown) produce oversized chunks. + """ + result = [] + secondary_splitter = _get_plain_splitter() + + for chunk in chunks: + if token_count(chunk) > CHUNK_SIZE: + # Split oversized chunk + sub_chunks = secondary_splitter.split_text(chunk) + result.extend(sub_chunks) + else: + result.append(chunk) + + return result + + +def chunk_text( + text: str, + content_type: Optional[ContentType] = None, + file_path: Optional[str] = None, +) -> List[str]: + """ + Split text into chunks using appropriate splitter for content type. + + Args: + text: The text to chunk + content_type: Optional explicit content type (auto-detected if not provided) + file_path: Optional file path for content type detection + + Returns: + List of text chunks, each approximately <= CHUNK_SIZE tokens + """ + if not text or not text.strip(): + return [] + + # Short text doesn't need chunking + text_tokens = token_count(text) + if text_tokens <= CHUNK_SIZE: + return [text] + + # Detect content type if not provided + if content_type is None: + content_type = detect_content_type(text, file_path) + + logger.debug(f"Chunking text with content type: {content_type.value}") + + # Select appropriate splitter + if content_type == ContentType.HTML: + splitter = _get_html_splitter() + # HTML splitter returns Document objects + docs = splitter.split_text(text) + chunks = [ + doc.page_content if hasattr(doc, "page_content") else str(doc) + for doc in docs + ] + elif content_type == ContentType.MARKDOWN: + splitter = _get_markdown_splitter() + # Markdown splitter returns Document objects + docs = splitter.split_text(text) + chunks = [ + doc.page_content if hasattr(doc, "page_content") else str(doc) + for doc in docs + ] + else: + # Plain text - use recursive splitter directly + splitter = _get_plain_splitter() + chunks = splitter.split_text(text) + + # Apply secondary chunking if needed (for HTML/Markdown that may produce large chunks) + if content_type in (ContentType.HTML, ContentType.MARKDOWN): + chunks = _apply_secondary_chunking(chunks) + + # Filter out empty chunks + chunks = [c.strip() for c in chunks if c and c.strip()] + + logger.debug(f"Created {len(chunks)} chunks from {text_tokens} tokens") + return chunks diff --git a/open_notebook/utils/context_builder.py b/open_notebook/utils/context_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..ed03712f95efbf4af6e3d4b21ffcc885826c7b92 --- /dev/null +++ b/open_notebook/utils/context_builder.py @@ -0,0 +1,495 @@ +""" +Generic ContextBuilder for the Open Notebook project. + +This module provides a flexible ContextBuilder class that can handle any parameters +and build context from sources, notebooks, insights, and notes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Optional + +from loguru import logger + +from open_notebook.domain.notebook import Note, Notebook, Source +from open_notebook.exceptions import DatabaseOperationError, NotFoundError + +from .token_utils import token_count + + +@dataclass +class ContextItem: + """Represents a single item in the context.""" + + id: str + type: Literal["source", "note", "insight"] + content: Dict[str, Any] + priority: int = 0 + token_count: Optional[int] = None + + def __post_init__(self): + """Calculate token count for the content if not provided.""" + if self.token_count is None: + content_str = str(self.content) + self.token_count = token_count(content_str) + + +@dataclass +class ContextConfig: + """Configuration for context building.""" + + sources: Optional[Dict[str, str]] = None # {source_id: inclusion_level} + notes: Optional[Dict[str, str]] = None # {note_id: inclusion_level} + include_insights: bool = True + include_notes: bool = True + max_tokens: Optional[int] = None + priority_weights: Optional[Dict[str, int]] = None # {type: weight} + + def __post_init__(self): + """Initialize default values.""" + if self.sources is None: + self.sources = {} + if self.notes is None: + self.notes = {} + if self.priority_weights is None: + self.priority_weights = {"source": 100, "note": 50, "insight": 75} + + +class ContextBuilder: + """ + Generic ContextBuilder that can handle any parameters and build context + from sources, notebooks, insights, and notes. + """ + + def __init__(self, **kwargs): + """ + Initialize ContextBuilder with flexible parameters. + + Supported parameters: + - source_id: str - Include specific source + - notebook_id: str - Include notebook content + - include_insights: bool - Include source insights + - include_notes: bool - Include notes + - context_config: ContextConfig - Custom context configuration + - max_tokens: int - Maximum token limit + - priority_order: List[str] - Custom priority order + """ + # Store all parameters for flexibility + self.params = kwargs + + # Extract commonly used parameters + self.source_id: Optional[str] = kwargs.get("source_id") + self.notebook_id: Optional[str] = kwargs.get("notebook_id") + self.include_insights: bool = kwargs.get("include_insights", True) + self.include_notes: bool = kwargs.get("include_notes", True) + self.max_tokens: Optional[int] = kwargs.get("max_tokens") + + # Context configuration + context_config_arg: Optional[ContextConfig] = kwargs.get("context_config") + self.context_config: ContextConfig + if context_config_arg is None: + self.context_config = ContextConfig( + include_insights=self.include_insights, + include_notes=self.include_notes, + max_tokens=self.max_tokens, + ) + else: + self.context_config = context_config_arg + + # Items storage + self.items: List[ContextItem] = [] + + logger.debug(f"ContextBuilder initialized with params: {list(kwargs.keys())}") + + async def build(self) -> Dict[str, Any]: + """ + Build context based on provided parameters. + + Returns: + Dict containing the built context with metadata + """ + try: + logger.info("Starting context building") + + # Clear existing items + self.items = [] + + # Build context based on parameters + if self.source_id: + await self._add_source_context(self.source_id) + + if self.notebook_id: + await self._add_notebook_context(self.notebook_id) + + # Process any additional custom parameters + await self._process_custom_params() + + # Apply post-processing + self.remove_duplicates() + self.prioritize() + + if self.max_tokens: + self.truncate_to_fit(self.max_tokens) + + # Format and return response + return self._format_response() + + except Exception as e: + logger.error(f"Error building context: {str(e)}") + raise DatabaseOperationError(f"Failed to build context: {str(e)}") + + async def _add_source_context( + self, source_id: str, inclusion_level: str = "insights" + ) -> None: + """ + Add source and its insights to context. + + Args: + source_id: ID of the source + inclusion_level: "insights", "full content", or "not in" + """ + if inclusion_level == "not in": + return + + try: + # Ensure source ID has table prefix + full_source_id = ( + source_id if source_id.startswith("source:") else f"source:{source_id}" + ) + + source = await Source.get(full_source_id) + if not source: + logger.warning(f"Source {source_id} not found") + return + + # Determine context size based on inclusion level + context_size: Literal["short", "long"] = ( + "long" if "full content" in inclusion_level else "short" + ) + source_context = await source.get_context(context_size=context_size) + + # Add source item + priority = (self.context_config.priority_weights or {}).get("source", 100) + item = ContextItem( + id=source.id or "", + type="source", + content=source_context, + priority=priority, + ) + self.add_item(item) + + # Add insights if requested and available + if self.include_insights and "insights" in inclusion_level: + insights = await source.get_insights() + for insight in insights: + insight_priority = (self.context_config.priority_weights or {}).get( + "insight", 75 + ) + insight_item = ContextItem( + id=insight.id or "", + type="insight", + content={ + "id": insight.id, + "source_id": source.id, + "insight_type": insight.insight_type, + "content": insight.content, + }, + priority=insight_priority, + ) + self.add_item(insight_item) + + logger.debug(f"Added source context for {source_id}") + + except NotFoundError: + logger.warning(f"Source {source_id} not found") + except Exception as e: + logger.error(f"Error adding source context for {source_id}: {str(e)}") + raise + + async def _add_notebook_context(self, notebook_id: str) -> None: + """ + Add notebook content based on context configuration. + + Args: + notebook_id: ID of the notebook + """ + try: + notebook = await Notebook.get(notebook_id) + if not notebook: + raise NotFoundError(f"Notebook {notebook_id} not found") + + # Process sources from context config or get all + config_sources = self.context_config.sources + if config_sources: + for source_id, status in config_sources.items(): + await self._add_source_context(source_id, status) + else: + # Default: get all sources with insights + sources = await notebook.get_sources() + for source in sources: + if source.id: + await self._add_source_context(source.id, "insights") + + # Process notes from context config or get all + if self.include_notes: + config_notes = self.context_config.notes + if config_notes: + for note_id, status in config_notes.items(): + if "not in" not in status: + await self._add_note_context(note_id, status) + else: + # Default: get all notes with short content + notes = await notebook.get_notes() + for note in notes: + if note.id: + await self._add_note_context(note.id, "full content") + + logger.debug(f"Added notebook context for {notebook_id}") + + except Exception as e: + logger.error(f"Error adding notebook context for {notebook_id}: {str(e)}") + raise + + async def _add_note_context( + self, note_id: str, inclusion_level: str = "full content" + ) -> None: + """ + Add note to context. + + Args: + note_id: ID of the note + inclusion_level: "full content" or "not in" + """ + if inclusion_level == "not in": + return + + try: + # Ensure note ID has table prefix + full_note_id = note_id if note_id.startswith("note:") else f"note:{note_id}" + + note = await Note.get(full_note_id) + if not note: + logger.warning(f"Note {note_id} not found") + return + + # Get note context + context_size: Literal["short", "long"] = ( + "long" if "full content" in inclusion_level else "short" + ) + note_context = note.get_context(context_size=context_size) + + # Add note item + priority = (self.context_config.priority_weights or {}).get("note", 50) + item = ContextItem( + id=note.id or "", type="note", content=note_context, priority=priority + ) + self.add_item(item) + + logger.debug(f"Added note context for {note_id}") + + except NotFoundError: + logger.warning(f"Note {note_id} not found") + except Exception as e: + logger.error(f"Error adding note context for {note_id}: {str(e)}") + + async def _process_custom_params(self) -> None: + """Process any additional custom parameters.""" + # Hook for future extensions - can be overridden in subclasses + # or used to process additional kwargs + for key, value in self.params.items(): + if key.startswith("custom_"): + logger.debug(f"Processing custom parameter: {key}={value}") + # Custom processing logic can be added here + + def add_item(self, item: ContextItem) -> None: + """ + Add a ContextItem to the builder. + + Args: + item: ContextItem to add + """ + self.items.append(item) + logger.debug(f"Added item {item.id} with priority {item.priority}") + + def prioritize(self) -> None: + """Sort items by priority (higher priority first).""" + self.items.sort(key=lambda x: x.priority, reverse=True) + logger.debug(f"Prioritized {len(self.items)} items") + + def truncate_to_fit(self, max_tokens: int) -> None: + """ + Remove items if total token count exceeds limit. + + Args: + max_tokens: Maximum allowed tokens + """ + if not max_tokens: + return + + total_tokens = sum(item.token_count or 0 for item in self.items) + + if total_tokens <= max_tokens: + logger.debug(f"Token count {total_tokens} within limit {max_tokens}") + return + + logger.info(f"Truncating from {total_tokens} to {max_tokens} tokens") + + # Remove items from the end (lowest priority) until under limit + current_tokens = total_tokens + removed_count = 0 + + while current_tokens > max_tokens and self.items: + removed_item = self.items.pop() + current_tokens -= removed_item.token_count or 0 + removed_count += 1 + + logger.info( + f"Removed {removed_count} items, final token count: {current_tokens}" + ) + + def remove_duplicates(self) -> None: + """Remove duplicate items based on ID.""" + seen_ids = set() + deduplicated_items = [] + + for item in self.items: + if item.id not in seen_ids: + deduplicated_items.append(item) + seen_ids.add(item.id) + + removed_count = len(self.items) - len(deduplicated_items) + self.items = deduplicated_items + + if removed_count > 0: + logger.debug(f"Removed {removed_count} duplicate items") + + def _format_response(self) -> Dict[str, Any]: + """ + Format the final response. + + Returns: + Formatted context response + """ + # Group items by type + sources = [] + notes = [] + insights = [] + + for item in self.items: + if item.type == "source": + sources.append(item.content) + elif item.type == "note": + notes.append(item.content) + elif item.type == "insight": + insights.append(item.content) + + # Calculate total tokens + total_tokens = sum(item.token_count or 0 for item in self.items) + + response = { + "sources": sources, + "notes": notes, + "insights": insights, + "total_tokens": total_tokens, + "total_items": len(self.items), + "metadata": { + "source_count": len(sources), + "note_count": len(notes), + "insight_count": len(insights), + "config": { + "include_insights": self.include_insights, + "include_notes": self.include_notes, + "max_tokens": self.max_tokens, + }, + }, + } + + # Add notebook_id if provided + if self.notebook_id: + response["notebook_id"] = self.notebook_id + + logger.info( + f"Built context with {len(self.items)} items, {total_tokens} tokens" + ) + + return response + + +# Convenience functions for common use cases + + +async def build_notebook_context( + notebook_id: str, + context_config: Optional[ContextConfig] = None, + max_tokens: Optional[int] = None, +) -> Dict[str, Any]: + """ + Build context for a notebook. + + Args: + notebook_id: ID of the notebook + context_config: Optional context configuration + max_tokens: Optional token limit + + Returns: + Built context + """ + builder = ContextBuilder( + notebook_id=notebook_id, context_config=context_config, max_tokens=max_tokens + ) + return await builder.build() + + +async def build_source_context( + source_id: str, include_insights: bool = True, max_tokens: Optional[int] = None +) -> Dict[str, Any]: + """ + Build context for a single source. + + Args: + source_id: ID of the source + include_insights: Whether to include insights + max_tokens: Optional token limit + + Returns: + Built context + """ + builder = ContextBuilder( + source_id=source_id, include_insights=include_insights, max_tokens=max_tokens + ) + return await builder.build() + + +async def build_mixed_context( + source_ids: Optional[List[str]] = None, + note_ids: Optional[List[str]] = None, + notebook_id: Optional[str] = None, + max_tokens: Optional[int] = None, +) -> Dict[str, Any]: + """ + Build context from mixed sources. + + Args: + source_ids: List of source IDs + note_ids: List of note IDs + notebook_id: Optional notebook ID + max_tokens: Optional token limit + + Returns: + Built context + """ + context_config = ContextConfig(max_tokens=max_tokens) + + # Configure sources + if source_ids: + context_config.sources = {sid: "insights" for sid in source_ids} + + # Configure notes + if note_ids: + context_config.notes = {nid: "full content" for nid in note_ids} + + builder = ContextBuilder( + notebook_id=notebook_id, context_config=context_config, max_tokens=max_tokens + ) + return await builder.build() diff --git a/open_notebook/utils/embedding.py b/open_notebook/utils/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..713e8706c401e5a043cb1507203c325f7207a967 --- /dev/null +++ b/open_notebook/utils/embedding.py @@ -0,0 +1,274 @@ +""" +Unified embedding utilities for Open Notebook. + +Provides centralized embedding generation with support for: +- Single text embedding (with automatic chunking and mean pooling for large texts) +- Batch text embedding (multiple texts with automatic batching) +- Mean pooling for combining multiple embeddings into one + +All embedding operations in the application should use these functions +to ensure consistent behavior and proper handling of large content. +""" + +import asyncio +import os +from typing import TYPE_CHECKING, List, Optional + +import numpy as np +from loguru import logger + +from .chunking import CHUNK_SIZE, ContentType, chunk_text +from .token_utils import token_count + + +def _get_embedding_batch_size() -> int: + """ + Read the embedding batch size from the environment. + + This is intentionally configurable because provider limits vary widely, and + CPU-only local embedding endpoints often need smaller batches than cloud APIs. + """ + raw = os.getenv("OPEN_NOTEBOOK_EMBEDDING_BATCH_SIZE", "50").strip() + try: + value = int(raw) + if value < 1: + raise ValueError + return value + except ValueError: + logger.warning( + "Invalid OPEN_NOTEBOOK_EMBEDDING_BATCH_SIZE='{}'; falling back to 50", + raw, + ) + return 50 + + +EMBEDDING_BATCH_SIZE = _get_embedding_batch_size() +EMBEDDING_MAX_RETRIES = 3 +EMBEDDING_RETRY_DELAY = 2 # seconds + +# Lazy import to avoid circular dependency: +# utils -> embedding -> models -> key_provider -> provider_config -> utils +if TYPE_CHECKING: + from open_notebook.ai.models import ModelManager + + +async def mean_pool_embeddings(embeddings: List[List[float]]) -> List[float]: + """ + Combine multiple embeddings into a single embedding using mean pooling. + + Algorithm: + 1. Normalize each embedding to unit length + 2. Compute element-wise mean + 3. Normalize the result to unit length + + This approach ensures the final embedding has the same properties as + individual embeddings (unit length) regardless of input count. + + Args: + embeddings: List of embedding vectors (each is a list of floats) + + Returns: + Single embedding vector (mean pooled and normalized) + + Raises: + ValueError: If embeddings list is empty or embeddings have different dimensions + """ + if not embeddings: + raise ValueError("Cannot mean pool empty list of embeddings") + + if len(embeddings) == 1: + # Single embedding - just normalize and return + arr = np.array(embeddings[0], dtype=np.float64) + norm = np.linalg.norm(arr) + if norm > 0: + arr = arr / norm + return arr.tolist() + + # Convert to numpy array + arr = np.array(embeddings, dtype=np.float64) + + # Verify all embeddings have same dimension + if arr.ndim != 2: + raise ValueError(f"Expected 2D array, got shape {arr.shape}") + + # Normalize each embedding to unit length + norms = np.linalg.norm(arr, axis=1, keepdims=True) + # Avoid division by zero + norms = np.where(norms > 0, norms, 1.0) + normalized = arr / norms + + # Compute mean + mean = np.mean(normalized, axis=0) + + # Normalize the result + mean_norm = np.linalg.norm(mean) + if mean_norm > 0: + mean = mean / mean_norm + + return mean.tolist() + + +async def generate_embeddings( + texts: List[str], command_id: Optional[str] = None +) -> List[List[float]]: + """ + Generate embeddings for multiple texts with automatic batching and retry. + + Texts are split into batches of EMBEDDING_BATCH_SIZE to avoid exceeding + provider payload limits. Each batch is retried up to EMBEDDING_MAX_RETRIES + times on transient failures. + + Args: + texts: List of text strings to embed + command_id: Optional command ID for error logging context + + Returns: + List of embedding vectors, one per input text + + Raises: + ValueError: If no embedding model is configured + RuntimeError: If embedding generation fails + """ + if not texts: + return [] + + # Lazy import to avoid circular dependency + from open_notebook.ai.models import model_manager + + embedding_model = await model_manager.get_embedding_model() + if not embedding_model: + raise ValueError( + "No embedding model configured. Please configure one in the Models section." + ) + + model_name = getattr(embedding_model, "model_name", "unknown") + + # Log text sizes for debugging + metrics: tuple[int, int, int, int] | None = None + + def _get_size_metrics() -> tuple[int, int, int, int]: + nonlocal metrics + if metrics is None: + token_sizes = [token_count(t) for t in texts] + metrics = ( + min(token_sizes), + max(token_sizes), + sum(token_sizes), + sum(len(t) for t in texts), + ) + return metrics + + logger.opt(lazy=True).debug( + "Generating embeddings for {} texts " + "(tokens: min={}, max={}, total={}; chars: total={})", + lambda: len(texts), + lambda: _get_size_metrics()[0], + lambda: _get_size_metrics()[1], + lambda: _get_size_metrics()[2], + lambda: _get_size_metrics()[3], + ) + + all_embeddings: List[List[float]] = [] + total_batches = (len(texts) + EMBEDDING_BATCH_SIZE - 1) // EMBEDDING_BATCH_SIZE + + for batch_idx in range(total_batches): + start = batch_idx * EMBEDDING_BATCH_SIZE + end = start + EMBEDDING_BATCH_SIZE + batch = texts[start:end] + + for attempt in range(1, EMBEDDING_MAX_RETRIES + 1): + try: + batch_embeddings = await embedding_model.aembed(batch) + all_embeddings.extend(batch_embeddings) + break + except Exception as e: + cmd_context = f" (command: {command_id})" if command_id else "" + if attempt < EMBEDDING_MAX_RETRIES: + logger.debug( + f"Embedding batch {batch_idx + 1}/{total_batches} " + f"attempt {attempt}/{EMBEDDING_MAX_RETRIES} failed " + f"using model '{model_name}'{cmd_context}: {e}. Retrying..." + ) + await asyncio.sleep(EMBEDDING_RETRY_DELAY) + else: + logger.debug( + f"Embedding batch {batch_idx + 1}/{total_batches} " + f"failed after {EMBEDDING_MAX_RETRIES} attempts " + f"using model '{model_name}'{cmd_context}: {e}" + ) + raise RuntimeError( + f"Failed to generate embeddings using model '{model_name}' " + f"(batch {batch_idx + 1}/{total_batches}, " + f"{len(batch)} texts): {e}" + ) from e + + logger.debug(f"Generated {len(all_embeddings)} embeddings in {total_batches} batch(es)") + return all_embeddings + + +async def generate_embedding( + text: str, + content_type: Optional[ContentType] = None, + file_path: Optional[str] = None, + command_id: Optional[str] = None, +) -> List[float]: + """ + Generate a single embedding for text, handling large content via chunking and mean pooling. + + For short text (<= CHUNK_SIZE tokens): + - Embeds directly and returns the embedding + + For long text (> CHUNK_SIZE tokens): + - Chunks the text using appropriate splitter for content type + - Embeds all chunks in batches + - Combines embeddings via mean pooling + + Args: + text: The text to embed + content_type: Optional explicit content type for chunking + file_path: Optional file path for content type detection + command_id: Optional command ID for error logging context + + Returns: + Single embedding vector (list of floats) + + Raises: + ValueError: If text is empty or no embedding model configured + RuntimeError: If embedding generation fails + """ + if not text or not text.strip(): + raise ValueError("Cannot generate embedding for empty text") + + text = text.strip() + text_tokens = token_count(text) + + # Check if chunking is needed + if text_tokens <= CHUNK_SIZE: + # Short text - embed directly + logger.debug(f"Embedding short text ({text_tokens} tokens) directly") + embeddings = await generate_embeddings([text], command_id=command_id) + return embeddings[0] + + # Long text - chunk and mean pool + logger.debug(f"Text exceeds chunk size ({text_tokens} tokens), chunking...") + + chunks = chunk_text(text, content_type=content_type, file_path=file_path) + + if not chunks: + raise ValueError("Text chunking produced no chunks") + + if len(chunks) == 1: + # Single chunk after splitting + embeddings = await generate_embeddings(chunks, command_id=command_id) + return embeddings[0] + + logger.debug(f"Embedding {len(chunks)} chunks and mean pooling") + + # Embed all chunks in batches + embeddings = await generate_embeddings(chunks, command_id=command_id) + + # Mean pool to get single embedding + pooled = await mean_pool_embeddings(embeddings) + + logger.debug(f"Mean pooled {len(embeddings)} embeddings into single vector") + return pooled diff --git a/open_notebook/utils/encryption.py b/open_notebook/utils/encryption.py new file mode 100644 index 0000000000000000000000000000000000000000..87cadd9d9149a9bf3717e9254efa80d80238be6c --- /dev/null +++ b/open_notebook/utils/encryption.py @@ -0,0 +1,198 @@ +""" +Field-level encryption for sensitive data using API keys. + +This module provides encryption/decryption for API keys stored in the database. +Fernet uses AES-128-CBC with HMAC-SHA256 for authenticated encryption. + +OPEN_NOTEBOOK_ENCRYPTION_KEY accepts **any string**. A Fernet key is derived +from it via SHA-256, so users can set a simple passphrase like +``OPEN_NOTEBOOK_ENCRYPTION_KEY=my-secret`` and it will work. + +Usage: + # Encrypt before storing + encrypted = encrypt_value(api_key) + + # Decrypt when reading + decrypted = decrypt_value(encrypted) +""" + +import base64 +import hashlib +import os +from pathlib import Path +from typing import Optional + +from cryptography.fernet import Fernet, InvalidToken +from loguru import logger + + +def get_secret_from_env(var_name: str) -> Optional[str]: + """ + Get a secret from environment, supporting Docker secrets pattern. + + Checks for VAR_FILE first (Docker secrets), then falls back to VAR. + + Args: + var_name: Base name of the environment variable (e.g., "OPEN_NOTEBOOK_ENCRYPTION_KEY") + + Returns: + The secret value, or None if not configured. + """ + # Check for _FILE variant first (Docker secrets) + file_path = os.environ.get(f"{var_name}_FILE") + if file_path: + try: + path = Path(file_path) + if path.exists() and path.is_file(): + secret = path.read_text().strip() + if secret: + logger.debug(f"Loaded {var_name} from file: {file_path}") + return secret + else: + logger.warning(f"{var_name}_FILE points to empty file: {file_path}") + else: + logger.warning(f"{var_name}_FILE path does not exist: {file_path}") + except Exception as e: + logger.error(f"Failed to read {var_name} from file {file_path}: {e}") + + # Fall back to direct environment variable + return os.environ.get(var_name) + + +def _get_or_create_encryption_key() -> str: + """ + Get encryption key from environment, requires explicit configuration. + + Priority: + 1. OPEN_NOTEBOOK_ENCRYPTION_KEY_FILE (Docker secrets) + 2. OPEN_NOTEBOOK_ENCRYPTION_KEY (environment variable) + + For production deployments, you MUST set OPEN_NOTEBOOK_ENCRYPTION_KEY explicitly! + + Returns: + Encryption key string. + + Raises: + ValueError: If no encryption key is configured. + """ + # First check environment/Docker secrets + key = get_secret_from_env("OPEN_NOTEBOOK_ENCRYPTION_KEY") + if key: + return key + + raise ValueError( + "OPEN_NOTEBOOK_ENCRYPTION_KEY is not set. " + "Set this environment variable to any secret string to enable " + "encrypted storage of API keys in the database." + ) + + +# Lazy-loaded encryption key: initialized on first use, not at import time. +# This prevents the entire app from crashing if the key is not yet configured +# when other modules import from this file. +_ENCRYPTION_KEY: Optional[str] = None + + +def _get_encryption_key() -> str: + """Get the encryption key, initializing lazily on first call.""" + global _ENCRYPTION_KEY + if _ENCRYPTION_KEY is None: + _ENCRYPTION_KEY = _get_or_create_encryption_key() + return _ENCRYPTION_KEY + + +def _ensure_fernet_key(key: str) -> str: + """ + Derive a valid Fernet key from an arbitrary string via SHA-256. + + Any string is accepted as input. The key is derived by hashing it with + SHA-256 and encoding the result as URL-safe base64. + """ + derived = hashlib.sha256(key.encode()).digest() + return base64.urlsafe_b64encode(derived).decode() + + +def get_fernet() -> Fernet: + """ + Get Fernet instance with the configured encryption key. + + Returns: + Fernet instance. + + Raises: + ValueError: If encryption key is not configured. + """ + return Fernet(_ensure_fernet_key(_get_encryption_key()).encode()) + + +def encrypt_value(value: str) -> str: + """ + Encrypt a string value using Fernet symmetric encryption. + + Args: + value: The plain text string to encrypt. + + Returns: + Base64-encoded encrypted string. + + Raises: + ValueError: If encryption is not configured. + """ + fernet = get_fernet() + return fernet.encrypt(value.encode()).decode() + + +def looks_like_fernet_token(s: str) -> bool: + """ + Check if string looks like a Fernet encrypted token. + + Fernet tokens are versioned (1 byte) + timestamp (8 bytes) + IV (16 bytes) + + ciphertext (variable, multiple of 16 with PKCS7 padding) + HMAC (32 bytes). + Minimum decoded size is 73 bytes (1+8+16+16+32) for the smallest payload. + """ + if len(s) < 100: # Base64 of 73 bytes = ~100 chars minimum + return False + try: + decoded = base64.urlsafe_b64decode(s) + # Fernet: version(1) + timestamp(8) + IV(16) + ciphertext(>=16) + HMAC(32) + # Minimum 73 bytes, ciphertext must be multiple of 16 (AES block size) + if len(decoded) < 73: + return False + ciphertext_len = len(decoded) - 1 - 8 - 16 - 32 + return ciphertext_len > 0 and ciphertext_len % 16 == 0 + except Exception: + return False + + +def decrypt_value(value: str) -> str: + """ + Decrypt a Fernet-encrypted string value. + + Handles graceful fallback for legacy unencrypted data. + + Args: + value: The encrypted string (or plain text for legacy data). + + Returns: + Decrypted plain text string, or original value if not encrypted. + + Raises: + ValueError: If encryption is not configured or if decryption fails + for what appears to be encrypted data (wrong key). + """ + fernet = get_fernet() + + try: + return fernet.decrypt(value.encode()).decode() + except InvalidToken: + if looks_like_fernet_token(value): + # Looks like encrypted data but failed to decrypt - likely wrong key + raise ValueError( + "Decryption failed: data appears to be encrypted but key is incorrect. " + "Check OPEN_NOTEBOOK_ENCRYPTION_KEY configuration." + ) + # Not a valid token - treat as legacy plaintext + return value + except Exception as e: + logger.error(f"Decryption failed: {e}") + raise ValueError(f"Decryption failed: {str(e)}") diff --git a/open_notebook/utils/error_classifier.py b/open_notebook/utils/error_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..b7ac4f757296dbdc2a02ded163d4e10fc7e55693 --- /dev/null +++ b/open_notebook/utils/error_classifier.py @@ -0,0 +1,103 @@ +""" +Error classification utility for LLM provider errors. + +Maps raw exceptions from AI providers/Esperanto/LangChain to user-friendly +error messages and appropriate exception types. +""" + +from loguru import logger + +from open_notebook.exceptions import ( + AuthenticationError, + ConfigurationError, + ExternalServiceError, + NetworkError, + OpenNotebookError, + RateLimitError, +) + +# Classification rules: (keywords, exception_class, user_message or None to pass through) +_CLASSIFICATION_RULES: list[tuple[list[str], type[OpenNotebookError], str | None]] = [ + # Authentication errors + ( + ["authentication", "unauthorized", "invalid api key", "invalid_api_key", "401"], + AuthenticationError, + "Authentication failed. Please check your API key in Settings -> Credentials.", + ), + # Rate limit errors + ( + ["rate limit", "rate_limit", "429", "too many requests", "quota exceeded"], + RateLimitError, + "Rate limit exceeded. Please wait a moment and try again.", + ), + # Model not found (pass through original message) + ( + ["model not found", "does not exist", "model_not_found"], + ConfigurationError, + None, + ), + # Configuration errors from provision.py (pass through) + ( + ["no model configured", "please go to settings"], + ConfigurationError, + None, + ), + # Network errors + ( + ["connecterror", "timeoutexception", "connection refused", "connection error", "timed out", "timeout"], + NetworkError, + "Could not connect to the AI provider. Please check your network connection and provider URL.", + ), + # Context length errors + ( + ["context length", "token limit", "maximum context", "context_length_exceeded", "max_tokens"], + ExternalServiceError, + "Content too large for the selected model. Try using a smaller selection or a model with a larger context window.", + ), + # Payload too large errors + ( + ["413", "payload too large", "request entity too large"], + ExternalServiceError, + "The request payload is too large for the AI provider. Try reducing the content size or using a different model.", + ), + # Provider availability errors + ( + ["500", "502", "503", "service unavailable", "overloaded", "internal server error"], + ExternalServiceError, + "The AI provider is temporarily unavailable. Please try again in a few minutes.", + ), +] + + +def classify_error(exception: BaseException) -> tuple[type[OpenNotebookError], str]: + """ + Classify a raw exception into a user-friendly error type and message. + + Args: + exception: Any exception from LLM providers/Esperanto/LangChain + + Returns: + Tuple of (exception_class, user_friendly_message) + """ + error_str = str(exception).lower() + error_type_name = type(exception).__name__.lower() + combined = f"{error_type_name}: {error_str}" + + for keywords, exc_class, message in _CLASSIFICATION_RULES: + for keyword in keywords: + if keyword in combined: + user_message = message if message is not None else _truncate(str(exception)) + return exc_class, user_message + + # Unclassified error - log for future improvement + logger.warning( + f"Unclassified LLM error ({type(exception).__name__}): {exception}" + ) + return ExternalServiceError, f"AI service error: {_truncate(str(exception))}" + + +def _truncate(text: str, max_length: int = 200) -> str: + """Truncate text to max_length to avoid leaking verbose internal details.""" + if len(text) <= max_length: + return text + return text[:max_length] + "..." diff --git a/open_notebook/utils/graph_utils.py b/open_notebook/utils/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8f44c4fd5cf6e076e1800491188edc0c5ef1f418 --- /dev/null +++ b/open_notebook/utils/graph_utils.py @@ -0,0 +1,23 @@ +import asyncio + +from langchain_core.runnables import RunnableConfig +from loguru import logger + + +async def get_session_message_count(graph, session_id: str) -> int: + """Get message count from LangGraph state, returns 0 on error.""" + try: + # Use sync get_state() in a thread (SqliteSaver doesn't support async) + thread_state = await asyncio.to_thread( + graph.get_state, + config=RunnableConfig(configurable={"thread_id": session_id}), + ) + if ( + thread_state + and thread_state.values + and "messages" in thread_state.values + ): + return len(thread_state.values["messages"]) + except Exception as e: + logger.warning(f"Could not fetch message count for session {session_id}: {e}") + return 0 diff --git a/open_notebook/utils/text_utils.py b/open_notebook/utils/text_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7ea14614d68a2a139fb78ebb16cb7db1b8e7bb --- /dev/null +++ b/open_notebook/utils/text_utils.py @@ -0,0 +1,145 @@ +""" +Text utilities for Open Notebook. +Extracted from main utils to avoid circular imports. +""" + +import re +import unicodedata +from typing import Tuple + +# Patterns for matching thinking content in AI responses +# Standard pattern: ... +THINK_PATTERN = re.compile(r"(.*?)", re.DOTALL) +# Pattern for malformed output: content (missing opening tag) +THINK_PATTERN_NO_OPEN = re.compile(r"^(.*?)", re.DOTALL) + + +def remove_non_ascii(text: str) -> str: + """Remove non-ASCII characters from text.""" + return re.sub(r"[^\x00-\x7F]+", "", text) + + +def remove_non_printable(text: str) -> str: + """Remove non-printable characters from text.""" + # Replace any special Unicode whitespace characters with a regular space + text = re.sub(r"[\u2000-\u200B\u202F\u205F\u3000]", " ", text) + + # Replace unusual line terminators with a single newline + text = re.sub(r"[\u2028\u2029\r]", "\n", text) + + # Remove control characters, except newlines and tabs + text = "".join( + char for char in text if unicodedata.category(char)[0] != "C" or char in "\n\t" + ) + + # Replace non-breaking spaces with regular spaces + text = text.replace("\xa0", " ").strip() + + # Keep letters (including accented ones), numbers, spaces, newlines, tabs, and basic punctuation + return re.sub(r"[^\w\s.,!?\-\n\t]", "", text, flags=re.UNICODE) + + +def parse_thinking_content(content: str) -> Tuple[str, str]: + """ + Parse message content to extract thinking content from tags. + + Handles both well-formed tags and malformed output where the opening + tag is missing but is present. + + Args: + content (str): The original message content + + Returns: + Tuple[str, str]: (thinking_content, cleaned_content) + - thinking_content: Content from within tags + - cleaned_content: Original content with blocks removed + + Example: + >>> content = "Let me analyze thisHere's my answer" + >>> thinking, cleaned = parse_thinking_content(content) + >>> print(thinking) + "Let me analyze this" + >>> print(cleaned) + "Here's my answer" + """ + # Input validation + if not isinstance(content, str): + return "", str(content) if content is not None else "" + + # Limit processing for very large content (100KB limit) + if len(content) > 100000: + return "", content + + # Find all well-formed thinking blocks + thinking_matches = THINK_PATTERN.findall(content) + + if thinking_matches: + # Join all thinking content with double newlines + thinking_content = "\n\n".join(match.strip() for match in thinking_matches) + + # Remove all ... blocks from the original content + cleaned_content = THINK_PATTERN.sub("", content) + + # Clean up extra whitespace + cleaned_content = re.sub(r"\n\s*\n\s*\n", "\n\n", cleaned_content).strip() + + return thinking_content, cleaned_content + + # Handle malformed output: content (missing opening tag) + # Some models like Nemotron output thinking without the opening tag + malformed_match = THINK_PATTERN_NO_OPEN.match(content) + if malformed_match: + thinking_content = malformed_match.group(1).strip() + # Remove the thinking content and tag + cleaned_content = content[malformed_match.end() :].strip() + return thinking_content, cleaned_content + + return "", content + + +def clean_thinking_content(content: str) -> str: + """ + Remove thinking content from AI responses, returning only the cleaned content. + + This is a convenience function for cases where you only need the cleaned + content and don't need access to the thinking process. + + Args: + content (str): The original message content with potential tags + + Returns: + str: Content with blocks removed and whitespace cleaned + + Example: + >>> content = "Let me think...Here's the answer" + >>> clean_thinking_content(content) + "Here's the answer" + """ + _, cleaned_content = parse_thinking_content(content) + return cleaned_content + + +def extract_text_content(content) -> str: + """Extract text from LLM response content. + + Handles both plain string responses and structured content formats + (e.g. Gemini's envelope format): + [{'type': 'text', 'text': '...', 'extras': {...}}] + + Args: + content: The content from an AI message, either a string or a list of parts. + + Returns: + The extracted text content as a string. + """ + if isinstance(content, str): + return content + if isinstance(content, list): + text_parts = [] + for part in content: + if isinstance(part, dict) and "text" in part: + text_parts.append(part["text"]) + elif isinstance(part, str): + text_parts.append(part) + return "".join(text_parts) + return str(content) diff --git a/open_notebook/utils/token_utils.py b/open_notebook/utils/token_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3791f0cfc3a987d0ef4d02fc138b138b28ffe9bb --- /dev/null +++ b/open_notebook/utils/token_utils.py @@ -0,0 +1,54 @@ +""" +Token utilities for Open Notebook. +Handles token counting and cost calculations for language models. +""" + +import os + +from open_notebook.config import TIKTOKEN_CACHE_DIR + +# Set tiktoken cache directory before importing tiktoken to ensure +# tokenizer encodings are cached persistently in the data folder +os.environ["TIKTOKEN_CACHE_DIR"] = TIKTOKEN_CACHE_DIR + + +def token_count(input_string: str) -> int: + """ + Count the number of tokens in the input string using the 'o200k_base' encoding. + + Args: + input_string (str): The input string to count tokens for. + + Returns: + int: The number of tokens in the input string. + """ + try: + import tiktoken + + encoding = tiktoken.get_encoding("o200k_base") + tokens = encoding.encode(input_string) + return len(tokens) + except (ImportError, OSError) as e: + # Fallback: handles ImportError (tiktoken not installed) AND network/OS + # errors such as urllib.error.URLError or ConnectionError raised in + # offline environments when the encoding file cannot be downloaded. + from loguru import logger + + logger.warning( + "tiktoken unavailable, falling back to word-count estimation: {}", e + ) + return int(len(input_string.split()) * 1.3) + + +def token_cost(token_count: int, cost_per_million: float = 0.150) -> float: + """ + Calculate the cost of tokens based on the token count and cost per million tokens. + + Args: + token_count (int): The number of tokens. + cost_per_million (float): The cost per million tokens. Default is 0.150. + + Returns: + float: The calculated cost for the given token count. + """ + return cost_per_million * (token_count / 1_000_000) diff --git a/open_notebook/utils/version_utils.py b/open_notebook/utils/version_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5591f37d993ce12cf638ba47875007f171af1b04 --- /dev/null +++ b/open_notebook/utils/version_utils.py @@ -0,0 +1,153 @@ +""" +Version utilities for Open Notebook. +Handles version comparison, GitHub version fetching, and package version management. +""" + +from importlib.metadata import PackageNotFoundError, version +from urllib.parse import urlparse + +import requests # type: ignore +import tomli +from packaging.version import parse as parse_version + + +async def get_version_from_github_async(repo_url: str, branch: str = "main") -> str: + """ + Fetch and parse the version from pyproject.toml in a public GitHub repository (async). + """ + from urllib.parse import urlparse + + import httpx + import tomli + + # Parse the GitHub URL + parsed_url = urlparse(repo_url) + if "github.com" not in parsed_url.netloc: + raise ValueError("Not a GitHub URL") + + # Extract owner and repo name from path + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) < 2: + raise ValueError("Invalid GitHub repository URL") + + owner, repo = path_parts[0], path_parts[1] + + # Construct raw content URL for pyproject.toml + raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/pyproject.toml" + + # Fetch the file with timeout using httpx + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get(raw_url) + response.raise_for_status() + + # Parse TOML content + pyproject_data = tomli.loads(response.text) + + # Try to find version + try: + # Check tool.poetry.version + version_str = pyproject_data["tool"]["poetry"]["version"] + except KeyError: + try: + # Check project.version + version_str = pyproject_data["project"]["version"] + except KeyError: + raise KeyError("Version not found in pyproject.toml") + + return version_str + +def get_version_from_github(repo_url: str, branch: str = "main") -> str: + """ + Fetch and parse the version from pyproject.toml in a public GitHub repository. + + Args: + repo_url (str): URL of the GitHub repository + branch (str): Branch name to fetch from (defaults to "main") + + Returns: + str: Version string from pyproject.toml + + Raises: + ValueError: If the URL is not a valid GitHub repository URL + requests.RequestException: If there's an error fetching the file + KeyError: If version information is not found in pyproject.toml + """ + # Parse the GitHub URL + parsed_url = urlparse(repo_url) + if "github.com" not in parsed_url.netloc: + raise ValueError("Not a GitHub URL") + + # Extract owner and repo name from path + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) < 2: + raise ValueError("Invalid GitHub repository URL") + + owner, repo = path_parts[0], path_parts[1] + + # Construct raw content URL for pyproject.toml + raw_url = ( + f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/pyproject.toml" + ) + + # Fetch the file with timeout + response = requests.get(raw_url, timeout=10) + response.raise_for_status() + + # Parse TOML content + pyproject_data = tomli.loads(response.text) + + # Try to find version in different possible locations + try: + # Check project.version first (poetry style) + version = pyproject_data["tool"]["poetry"]["version"] + except KeyError: + try: + # Check project.version (standard style) + version = pyproject_data["project"]["version"] + except KeyError: + raise KeyError("Version not found in pyproject.toml") + + return version + + +def get_installed_version(package_name: str) -> str: + """ + Get the version of an installed package. + + Args: + package_name (str): Name of the installed package + + Returns: + str: Version string of the installed package + + Raises: + PackageNotFoundError: If the package is not installed + """ + try: + return version(package_name) + except PackageNotFoundError: + raise PackageNotFoundError(f"Package '{package_name}' not found") + + +def compare_versions(version1: str, version2: str) -> int: + """ + Compare two semantic versions. + + Args: + version1 (str): First version string + version2 (str): Second version string + + Returns: + int: -1 if version1 < version2 + 0 if version1 == version2 + 1 if version1 > version2 + """ + v1 = parse_version(version1) + v2 = parse_version(version2) + + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + else: + return 0 diff --git a/prompts/CLAUDE.md b/prompts/CLAUDE.md new file mode 100644 index 0000000000000000000000000000000000000000..976a1a75b7e5fa200e14871b516ec1db018f8f2c --- /dev/null +++ b/prompts/CLAUDE.md @@ -0,0 +1,190 @@ +# Prompts Module + +Jinja2 prompt templates for multi-provider AI workflows in Open Notebook. + +## Purpose + +Centralized prompt repository using `ai_prompter` library to: +1. Separate prompt engineering from Python application logic +2. Provide reusable Jinja2 templates with variable injection +3. Support multi-stage prompt chains (orchestrated by LangGraph workflows) +4. Ensure consistency across similar workflows (chat, search, content generation) + +## Architecture Overview + +**Template Organization by Workflow**: +- **`ask/`**: Multi-stage search synthesis (entry → query_process → final_answer) +- **`chat/`**: Conversational agent with notebook context (system prompt only) +- **`source_chat/`**: Source-focused chat with insight injection (system prompt only) +- **`podcast/`**: Podcast generation pipeline (outline → transcript) + +**Rendering Pattern** (all workflows): +```python +from ai_prompter import Prompter + +# Load template + render with variables +system_prompt = Prompter(prompt_template="ask/entry", parser=parser).render( + data=state +) + +# Then invoke LLM +model = await provision_langchain_model(system_prompt, ...) +response = await model.ainvoke(system_prompt) +``` + +See detailed workflow integration in `open_notebook/graphs/CLAUDE.md` for how each template fits into chat.py, ask.py, source_chat.py. + +## Prompt Engineering Patterns + +### 1. Multi-Stage Chain (Ask Workflow) + +Three-template chain for intelligent search: + +``` +entry.jinja (user question → search strategy) + ↓ +query_process.jinja (run each search, generate sub-answer) + ↓ (multiple parallel) +final_answer.jinja (synthesize all results into final response) +``` + +**Key pattern**: `entry.jinja` generates JSON-structured reasoning (via PydanticOutputParser). Each `query_process.jinja` invocation receives one search term + retrieved results. `final_answer.jinja` combines all answers with proper source citation. + +### 2. Conditional Variable Injection (Podcast Workflow) + +Templates accept optional variables for context assembly: + +```jinja +{% if notebook %} +# PROJECT INFORMATION +{{ notebook }} +{% endif %} + +{% if context %} +# CONTEXT +{{ context }} +{% endif %} +``` + +Enabled by Jinja2's conditional blocks. Critical for podcast outline (handles list or string context) and source_chat (injects variable notebook/insight data). + +### 3. Repeated Emphasis on Citation Format (Ask & Chat) + +All response-generating templates emphasize source citation rules: +- Document ID syntax: `[source:id]`, `[note:id]`, `[insight:id]` +- "Do not make up document IDs" repeated multiple times +- Example citations provided inline + +**Rationale**: LLMs naturally hallucinate citations without explicit guidance; repetition + examples reduce hallucination. + +### 4. Format Instructions Delegation + +Templates accept external `{{ format_instructions }}` variable: + +```jinja +# OUTPUT FORMATTING +{{ format_instructions }} +``` + +Allows caller to inject JSON schema, XML format, or other output constraints without modifying template. Decouples prompt from output format evolution. + +### 5. JSON Output with Extended Thinking Support + +Podcast templates include extended thinking pattern: + +```jinja +IMPORTANT OUTPUT FORMAT: +- If you use extended thinking with tags, put ALL your reasoning inside tags +- Put the final JSON output OUTSIDE and AFTER any tags +``` + +Guides models with extended thinking capability to separate reasoning from output (cleaner parsing downstream). + +## File Catalog + +**`ask/` - Search Synthesis Pipeline**: +- **entry.jinja**: Analyzes user question, generates search strategy with JSON output (term + instructions per search) +- **query_process.jinja**: Accepts one search term + retrieved results, generates sub-answer with citations +- **final_answer.jinja**: Combines all sub-answers into coherent final response, enforces source citation + +**`chat/` - Conversational Agent**: +- **system.jinja**: Single system prompt for general chat. Uses conditional blocks for optional notebook context. Emphasizes citation format. + +**`source_chat/` - Source-Focused Chat**: +- **system.jinja**: Single system prompt for source-specific discussion. Injects source metadata (ID, title, topics) + selected context. Conditional blocks for optional notebook/context data. + +**`podcast/` - Podcast Generation**: +- **outline.jinja**: Takes briefing + content + speaker profiles (list support via Jinja2 for-loop). Generates JSON outline with segments (name, description, size). +- **transcript.jinja**: Takes outline + segment index + optional existing transcript. Generates JSON dialogue array (speaker name + dialogue). Iterates speakers with for-loop. + +## Key Dependencies + +- **ai_prompter**: Prompter class for Jinja2 template rendering with optional OutputParser binding +- **Jinja2** (transitive via ai_prompter): Template syntax (if/for, filters, variable interpolation) +- **No external AI calls**: Templates are pure text; LLM invocation happens in calling code (graphs/) + +## How to Add New Template + +1. **Create subdirectory** in `prompts/` matching workflow name (e.g., `prompts/new_workflow/`) +2. **Define .jinja file(s)** with Jinja2 syntax: + - Use `{{ variable_name }}` for scalar injection + - Use `{% if condition %} ... {% endif %}` for optional sections + - Use `{% for item in list %} ... {% endfor %}` for iteration +3. **Document template variables** as inline comments (follow existing templates) +4. **Reference in calling code** (graphs/): + ```python + from ai_prompter import Prompter + prompt = Prompter(prompt_template="new_workflow/template_name").render(data=context_dict) + ``` +5. **If structured output needed**: Pass `parser=PydanticOutputParser(...)` to Prompter +6. **Document in graphs/CLAUDE.md** how new template fits into workflow chain + +## Important Quirks & Gotchas + +1. **Template path syntax**: Uses forward slashes without `.jinja` extension in Prompter. `"ask/entry"` maps to `/prompts/ask/entry.jinja` +2. **Variable key convention**: All data passed as `data=dict` arg to `.render()`. Template accesses variables directly (e.g., `{{ question }}`). Ensure dict keys match template variable names. +3. **OutputParser binding**: When using PydanticOutputParser, Prompter auto-injects `{{ format_instructions }}` into template. If template doesn't have this placeholder, parser is ignored. +4. **Jinja2 whitespace sensitivity**: Template indentation doesn't affect output, but raw newlines do. Use explicit `\n` or trim filters if output formatting matters. +5. **Conditional blocks are loose**: Jinja2 if-condition evaluates any truthy value (non-empty string, list, dict). `{% if variable %}` is False for empty string/"" but True for any non-empty content. +6. **For-loop list assumption**: Templates using `{% for item in list %}` don't validate list type. If caller passes string instead of list, iteration happens character-by-character (bug risk). +7. **No template composition/inheritance**: Templates are flat (no `{% extends %}` or `{% include %}`). Each workflow keeps templates independent to avoid coupling. +8. **Citation ID format is caller's responsibility**: Templates emphasize citation rules but don't validate. If caller returns wrong ID format, template can't catch it upstream. +9. **Parser extraction happens post-render**: OutputParser.parse() is called AFTER `.render()` returns string. If template has syntax errors, render fails before parsing logic runs. +10. **Template cache**: Prompter likely caches loaded templates. File edits require app restart if using cached instance. + +## Testing Patterns + +**Manual render test**: +```python +from ai_prompter import Prompter + +prompt = Prompter(prompt_template="ask/entry").render( + data={"question": "What is RAG?"} +) +print(prompt) # Inspect Jinja2 output before sending to LLM +``` + +**With parser**: +```python +from pydantic import BaseModel +from langchain_core.output_parsers.pydantic import PydanticOutputParser + +class Strategy(BaseModel): + reasoning: str + searches: list + +parser = PydanticOutputParser(pydantic_object=Strategy) +prompt = Prompter(prompt_template="ask/entry", parser=parser).render( + data={"question": "..."} +) +# prompt now includes {{ format_instructions }} substitution +``` + +**Integration test** (invoke full graph): +See `open_notebook/graphs/ask.py` for how entry.jinja is invoked inside ask_graph workflow. + +## Reference Documentation + +- **Jinja2 syntax guide**: See existing templates for for-loop, if-conditional, variable interpolation patterns +- **Graph integration**: `open_notebook/graphs/CLAUDE.md` documents which template is used in which workflow +- **Sub-directory CLAUDE.md files**: `ask/CLAUDE.md`, `chat/CLAUDE.md`, `podcast/CLAUDE.md` (if created) provide template-specific implementation notes diff --git a/prompts/ask/entry.jinja b/prompts/ask/entry.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a97f03a6da0552c9d97ce4dacb4a477f8b01eb6e --- /dev/null +++ b/prompts/ask/entry.jinja @@ -0,0 +1,45 @@ +# SYSTEM ROLE + +You are a cognitive study assistant that helps users research and learn by engaging in focused discussions about documents in their workspace. + +The first step in the process is receiving the user's question and formulating a research strategy to find the most relevant information. + +# YOUR JOB + +Based on the user question, you need to analyze the key concepts and terms to determine the appropriate search strategy. + +Step 1: develop your search strategy (reasoning) +Step 2: formulate your search queries (searches) + +Return both the reasoning and searches as a JSON object, like in the EXAMPLE below. + +# EXAMPLE + +User: Can you tell me more about the concept of "RAG" and how it can be applied to generate answers to user questions via LLM? + +Your answer could be something like: + +```json +{ + "reasoning": "The user is asking about the concept of RAG and its application in generating answers to user questions via LLM. I should search for documents related to RAG, retrieval augmented generation, and vector search to provide a comprehensive response.", + "searches": [ + { "term": "RAG", "instructions": "Describe the concept and utility of RAG." }, + { "term": "Retrieval Augmented Generation", "instructions": "Describe the concept and utility of RAG." }, + { "term": "Vector Search", "instructions": "Describe how RAG utilizes vector search." } + ] +} +``` + +# OUTPUT FORMATTING + +{{format_instructions}} + +- Do not include any text other than the JSON object +- Do not include ```json``` in the response + +# USER QUESTION + +{{question}} + +# ANSWER + diff --git a/prompts/ask/final_answer.jinja b/prompts/ask/final_answer.jinja new file mode 100644 index 0000000000000000000000000000000000000000..9c8b2d0fee5769e340bda3dcf66e8404b171d192 --- /dev/null +++ b/prompts/ask/final_answer.jinja @@ -0,0 +1,40 @@ +# SYSTEM ROLE + +You are a cognitive study assistant that helps users research and learn by engaging in focused discussions about documents in their workspace. + +You are responsible for the last step of the process, which is to provide the final answer to the user's question. You should provide accurate, factual responses based on the available documents and knowledge, while avoiding speculation or making up information. If you are unsure about something, acknowledge the uncertainty rather than guessing. + +# QUESTION + +This is the question originally made by the user: + +{{question}} + +# REASONS + +Based on the question, you derived the following reasonsing and search strategies: + +{{strategy}} + +# RESULTS + +Here are the answers you received for each of your queries. + +{{answers}} + +# YOUR JOB + +Based on the user question, the context and the retrieved answers, please formulate a final response to the user. + +# CITING SOURCES + +It's very important that your response contains references to the searched documents so the user can follow-up and read more about the topic. The way you do that is by adding the id of the specific document in between brackets like this: [document_id]. The references will be present on all the answers you have been provided. + +## IMPORTANT + +- Do not make up documents or document ids. Only use the ids of the documents that you can see on the answers you received. +- The ID is composed of the type of document and a random string, such as "source:randomstring", "note:randomstring", or "insight:randomstring". There are various types of documents, including notes, insights, and sources. **Always use the complete ID exactly as it is provided, including its type prefix. Do not add, remove, or modify any part of the ID.** +- **Use document IDs exactly as they are returned in the answers. Do not add any prefixes or modify them in any way.** + +# YOUR ANSWER + diff --git a/prompts/ask/query_process.jinja b/prompts/ask/query_process.jinja new file mode 100644 index 0000000000000000000000000000000000000000..e787fab74d61e572ba284e0539a54c673d71f4e5 --- /dev/null +++ b/prompts/ask/query_process.jinja @@ -0,0 +1,54 @@ +# SYSTEM ROLE + +You are a research assistant that helps users research and learn by engaging in focused discussions about documents in their workspace. + +# QUESTION + +This is the question originally made by the user: + +{{question}} + +# SEARCH STRATEGY + +The main answer agent has developed the following search strategy to find the most relevant information: + +{{term}} + +And provided you with the following instructions to formulate the answer: + +{{instructions}} + +# YOUR JOB + +Based on the user question, the context and the retrieved results, please formulate the appropriate answer. + +# RESULTS + +{{results}} + +# CITING SOURCES + +It's very important that your response contains references to the searched documents so the user can follow-up and read more about the topic. The way you do that is by adding the id of the specific document in between brackets like this: [document_id]. + +## EXAMPLE + +User: Can you tell me more about the concept of "Deep Learning"? + +Assistant: Deep learning is a subset of machine learning in artificial intelligence (AI) that enables networks to learn unsupervised from unstructured or unlabeled data. [note:iuiodadalknda]. It can also be categorized into three main types: supervised, unsupervised, and reinforcement learning. [insight:adadadadadadad]. + +Please note, "note:iuiodadalknda" and "insight:adadadadadadad" are examples of document IDs with different prefixes. You should not make up document IDs or copy the IDs from this example. You should use the IDs of the documents that you have access to through the search tool. + +## IMPORTANT + +- Do not make up documents or document ids. Only use the ids of the documents that you have access through the query you made. +- The ID is composed of the type of document and a random string, such as "source:randomstring", "note:randomstring", or "insight:randomstring". There are various types of documents, including notes, insights, and sources. **Always use the complete ID exactly as it is provided, including its type prefix. Do not add, remove, or modify any part of the ID.** +- Do not assume or change the type prefix of any document ID. If a document ID is "note:xyz", use it exactly as "note:xyz". Do not change it to "source:xyz" or any other variation. +- **Use document IDs exactly as they are returned from the search tool. Do not add any prefixes or modify them in any way.** + +## IDs PROVIDED IN THIS QUERY + +You have been given the following content ids to work from: {{ids}} +So, if you are citing some document, it should be one of these. + +# YOUR ANSWER + diff --git a/prompts/chat/system.jinja b/prompts/chat/system.jinja new file mode 100644 index 0000000000000000000000000000000000000000..f56e75be5bb35a9ea59a7cc0964ac674a819f5ad --- /dev/null +++ b/prompts/chat/system.jinja @@ -0,0 +1,43 @@ +# SYSTEM ROLE +You are a cognitive study assistant that helps users research and learn by engaging in focused discussions about documents in their workspace. You have access to project context and can analyze documents in detail using specialized tools. + +# CAPABILITIES +- Access to project information and selected documents (CONTEXT) +- Can engage in natural dialogue while maintaining academic rigor + +# YOUR OPERATING METHOD +Whenever a user asks you a question, you need to identify the query context and the user intent. The user might be continuing a previous conversation or asking a new question. Looking at the CONTEXT will probably give you a hint of what the user is looking for. Once you identify the user intent, formulate your answer accordingly paying attention to the CITING INSTRUCTIONS below. + +{% if notebook %} +# PROJECT INFORMATION + +**Name:** {{notebook.name}} +**Description:** {{notebook.description}} +{% endif %} + +{% if context %} +# CONTEXT + +The user has selected this context to help you with your response: + +{{context}} +{% endif %} + +# CITING INSTRUCTIONS + +If your answer is based off of any item in the context, it's very important that your response contains references to the searched documents so the user can follow-up and read more about the topic. The way you do that is by adding the id of the specific document in between brackets like this: [document_id]. + +## EXAMPLE + +User: Can you tell me more about the concept of "Deep Learning"? + +Assistant: Deep learning is a subset of machine learning in artificial intelligence (AI) that enables networks to learn unsupervised from unstructured or unlabeled data. [note:iuiodadalknda]. It can also be categorized into three main types: supervised, unsupervised, and reinforcement learning. [insight:adadadadadadad]. + +Please note, "note:iuiodadalknda" and "insight:adadadadadadad" are examples of document IDs with different prefixes. You should not make up document IDs or copy the IDs from this example. You should use the IDs of the documents that you have access to through the search tool. + +## IMPORTANT + +- Do not make up documents or document ids. Only use the ids of the documents that you have access through the query you made. +- The ID is composed of the type of document and a random string, such as "source:randomstring", "note:randomstring", or "insight:randomstring". There are various types of documents, including notes, insights, and sources. **Always use the complete ID exactly as it is provided, including its type prefix. Do not add, remove, or modify any part of the ID.** +- Do not assume or change the type prefix of any document ID. If a document ID is "note:xyz", use it exactly as "note:xyz". Do not change it to "source:xyz" or any other variation. +- **Use document IDs exactly as they are returned from the search tool. Do not add any prefixes or modify them in any way.** diff --git a/prompts/podcast/outline.jinja b/prompts/podcast/outline.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ae13a7fee4e9c94b413e3273d29738fa381b929 --- /dev/null +++ b/prompts/podcast/outline.jinja @@ -0,0 +1,83 @@ +You are an AI assistant specialized in creating podcast outlines. Your task is to create a detailed outline for a podcast episode based on a provided briefing. The outline you create will be used to generate the podcast transcript. + +Here is the briefing for the podcast episode: + +{{ briefing }} + + +The user has provided content to be used as the context for this podcast episode: + +{% if context is string %} +{{ context }} +{% else %} +{% for item in context %} + +{{ item }} + +{% endfor %} +{% endif %} + + +The podcast will feature the following speakers: + +{% for speaker in speakers %} +- **{{ speaker.name }}**: {{ speaker.backstory }} + Personality: {{ speaker.personality }} +{% endfor %} + + +Please create an outline based on this briefing. Your outline should consist of {{ num_segments }} main segments for the podcast episode, along with a description of each segment. Follow these guidelines: + +1. Read the briefing carefully and identify the main topics and themes. +2. Create {{ num_segments }} distinct segments that cover the entire scope of the briefing. +3. For each segment, provide a clear and concise name that reflects its content. +4. Write a detailed description for each segment, explaining what will be discussed and provide suggestions of topics according to the context given. The writer will use your suggestion to design the dialogs. +5. Consider the speaker personalities and backstories when planning segments - match content to speaker expertise. +6. Ensure that the segments flow logically from one to the next. +7. This is a whole podcast so no need to reintroduce speakers or topics on each segment. Segments are just markers for us to know to change the topics, nothing else. +8. Include an introduction segment at the beginning and a conclusion or wrap-up segment at the end. + +Format your outline using the following structure: + +```json +{ + "segments": [ + { + "name": "[Segment Name]", + "description": "[Description of the segment content]", + "size": "short" + }, + { + "name": "[Segment Name]", + "description": "[Description of the segment content]", + "size": "medium" + }, + { + "name": "[Segment Name]", + "description": "[Description of the segment content]", + "size": "long" + }, + ... + ] +} +``` + +Formatting instructions: +{{ format_instructions}} + +Additional tips: +- Make sure the segment names are catchy and informative. +- In the descriptions, include key points or questions that will be addressed in each segment. +- Consider the target audience mentioned in the briefing when crafting your outline. +- If the briefing mentions a guest, include segments for introducing the guest and featuring their expertise. +- The size of the segment should be short, medium or long. Think about the content of the segment and how important it is to the episode. + +IMPORTANT OUTPUT FORMAT: +- If you use extended thinking with tags, put ALL your reasoning inside tags +- Put the final JSON output OUTSIDE and AFTER any tags +- Do NOT wrap the JSON in ```json code blocks - return the raw JSON object only +- Example correct format: + Let me analyze the briefing... + {"segments": [...]} + +Please provide your outline now, following the format and guidelines provided above. diff --git a/prompts/podcast/transcript.jinja b/prompts/podcast/transcript.jinja new file mode 100644 index 0000000000000000000000000000000000000000..974b3073919171279cb132715b410b881cc1ae39 --- /dev/null +++ b/prompts/podcast/transcript.jinja @@ -0,0 +1,134 @@ +You are an AI assistant specialized in creating podcast transcripts. +Your task is to generate a transcript for a specific segment of a podcast episode based on a provided briefing and outline. +The transcript will be used to generate podcast audio. Follow these instructions carefully: + +First, review the briefing for the podcast episode: + +{{ briefing }} + + +The user has provided content to be used as the context for this podcast episode: + +{% if context is string %} +{{ context }} +{% else %} +{% for item in context %} + +{{ item }} + +{% endfor %} +{% endif %} + + +The podcast features the following speakers: + +{% for speaker in speakers %} +- **{{ speaker.name }}**: {{ speaker.backstory }} + Personality: {{ speaker.personality }} +{% endfor %} + + +Next, examine the outline produced by our director: + +{{ outline }} + + +{% if transcript %} +Here is the current transcript so far: + +{{ transcript }} + +{% endif %} + +{% if is_final %} +{% if speakers|length == 1 %} +This is the final segment of the podcast. Make sure to wrap up the presentation and provide a conclusion. +{% else %} +This is the final segment of the podcast. Make sure to wrap up the conversation and provide a conclusion. +{% endif %} +{% endif %} + + +You will focus on creating the dialogue for the following segment ONLY: + +{{ segment }} + + +{% if speakers|length == 1 %} +IMPORTANT: This is a SOLO podcast with only ONE speaker ({{ speaker_names[0] }}). Do NOT invent or add any other speakers. +All dialogue entries must use "{{ speaker_names[0] }}" as the speaker name. + +Follow these format requirements strictly: + - Use ONLY the speaker name "{{ speaker_names[0] }}" for all dialogue entries. + - Do NOT create or invent any additional speakers. + - Stick to the segment, do not go further than what's requested. Other agents will do the rest of the podcast. + - The transcript must have at least {{ turns }} dialogue segments from the speaker. + - The speaker should present the content in an engaging, educational manner. +{% else %} +Follow these format requirements strictly: + - Use the actual speaker names ({{ speaker_names|join(', ') }}) to denote speakers. + - Choose which speaker should speak based on their personality, backstory, and the content being discussed. + - Stick to the segment, do not go further than what's requested. Other agents will do the rest of the podcast. + - The transcript must have at least {{ turns }} turns of messages between the speakers. + - Each speaker should contribute meaningfully based on their expertise and personality. +{% endif %} + + +```json +{ + "transcript": [ + { + "speaker": "[Actual Speaker Name]", + "dialogue": "[Speaker's dialogue based on their personality and expertise]" + }, + ... + ] +} +``` + +Formatting instructions: +{{ format_instructions}} + + +{% if speakers|length == 1 %} +Guidelines for creating the transcript: + - Ensure the presentation flows naturally and covers all points in the outline. + - Ensure you return the root "transcript" key in your response. + - Make the content sound engaging and educational. + - Include relevant details from the briefing. + - Break up the content into digestible segments with natural transitions. + - Use appropriate transitions between topics. + - Match the speaker's dialogue to their personality and expertise. + - This is a whole podcast so no need to reintroduce the speaker or topics on each segment. Segments are just markers for us to know to change the topics, nothing else. + - CRITICAL: There is only ONE speaker. Use ONLY: {{ speaker_names[0] }}. Do NOT invent additional speakers. +{% else %} +Guidelines for creating the transcript: + - Ensure the conversation flows naturally and covers all points in the outline. + - Ensure you return the root "transcript" key in your response. + - Make the dialogue sound conversational and engaging. + - Include relevant details from the briefing. + - Avoid long monologues; keep exchanges between speakers balanced. + - Use appropriate transitions between topics. + - Match each speaker's dialogue to their personality and expertise. + - Choose speakers strategically based on who would naturally contribute to each topic. + - This is a whole podcast so no need to reintroduce speakers or topics on each segment. Segments are just markers for us to know to change the topics, nothing else. + - IMPORTANT: Only use the provided speaker names: {{ speaker_names|join(', ') }} +{% endif %} + +IMPORTANT OUTPUT FORMAT: +- If you use extended thinking with tags, put ALL your reasoning inside tags +- Put the final JSON output OUTSIDE and AFTER any tags +- Do NOT wrap the JSON in ```json code blocks - return the raw JSON object only +- Example correct format: + Let me plan the dialogue... + {"transcript": [...]} + +When you're ready, provide the transcript. +{% if speakers|length == 1 %} +Remember, you are creating a realistic solo podcast presentation based on the given information. +Make it informative, engaging, and natural-sounding while adhering to the format requirements. +There is only ONE speaker - do not add any other speakers. +{% else %} +Remember, you are creating a realistic podcast conversation based on the given information. +Make it informative, engaging, and natural-sounding while adhering to the format requirements. +{% endif %} diff --git a/prompts/source_chat/system.jinja b/prompts/source_chat/system.jinja new file mode 100644 index 0000000000000000000000000000000000000000..d7494061c29809b5d0a1a30641f020ea4ff04c2f --- /dev/null +++ b/prompts/source_chat/system.jinja @@ -0,0 +1,63 @@ +# SYSTEM ROLE +You are a specialized research assistant focused on helping users deeply understand and analyze a specific source document. You have access to the source content and its generated insights, and you can engage in detailed discussions about this material. + +# CAPABILITIES +- Deep analysis of the specific source document and its content +- Access to AI-generated insights and analysis from this source +- Can answer questions, explain concepts, and provide detailed analysis +- Can reference specific sections and insights from the source + +# YOUR OPERATING METHOD +When a user asks you a question, analyze both the source content and the available insights to provide comprehensive, accurate responses. Focus on helping the user understand the material, make connections, and explore ideas related to this specific source. + +{% if source %} +# SOURCE INFORMATION + +**Source ID:** {{ source.id }} +**Title:** {{ source.title or "No title" }} + +{% if source.topics %} +**Topics:** {{ source.topics | join(", ") }} +{% endif %} +{% endif %} + +{% if context %} +# SOURCE CONTEXT + +{{ context }} +{% endif %} + +# CITING INSTRUCTIONS + +When referencing information from the source or its insights, always include citations using the document IDs. This helps users track the specific content you're referencing. + +## Citation Format +- For source content: [{{ source.id if source else "source:id" }}] +- For insights: [insight_id] (use the specific insight ID) + +## EXAMPLE + +User: What are the main themes in this document? +Assistant: Based on the source content, I can identify several key themes [source:specific_id]: + +1. **Theme 1**: The document discusses X, which appears in several insights [insight:specific_insight_id] +2. **Theme 2**: Another important concept is Y, as shown in [source:specific_id] + +Each theme is supported by specific insights and passages from the source material. + +## IMPORTANT + +- **Do not make up document IDs or insight IDs.** Only use the IDs that are actually available in the context. +- **Use complete IDs exactly as provided**, including their type prefix (source:, insight:, etc.) +- **Always reference specific content** when citing to help users locate the information +- **Focus on the specific source** - this chat is dedicated to understanding this particular document +- **Leverage insights** to provide deeper analysis beyond just the raw content + +# CONVERSATION FOCUS + +This conversation is specifically about the source document provided in the context. Help users: +- Understand complex concepts within the document +- Make connections between different parts of the source +- Explore implications and deeper meanings +- Ask follow-up questions to deepen their understanding +- Navigate through the available insights for different perspectives diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ebfc8e9ac7b6805e6ad81a47e2ad53190215c7e8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,66 @@ +# ============================================================================= +# COMPLETE DEPENDENCIES FOR OPEN NOTEBOOK +# Generated: January 2026 +# ============================================================================= + +# Core Web Framework +fastapi>=0.104.0 +uvicorn[standard]>=0.24.0 +pydantic>=2.9.2 +python-dotenv>=1.0.1 +python-multipart>=0.0.6 + +# Logging +loguru>=0.7.2 + +# Database +surrealdb>=1.0.4 +surreal-commands>=1.2.0 + +# LLM & AI Orchestration Core +langchain>=0.3.3 +langgraph>=0.2.38 +tiktoken>=0.8.0 +langgraph-checkpoint-sqlite>=2.0.0 +langchain-community>=0.3.3 + +# LangChain Integrations +langchain-openai>=0.2.3 +langchain-anthropic>=0.2.3 +langchain-ollama>=0.2.0 +langchain-google-genai>=2.1.10 +langchain-groq>=0.2.1 +langchain_mistralai>=0.2.1 +langchain_deepseek>=0.1.3 +langchain-google-vertexai>=2.0.28 +langchain-text-splitters>=0.3.0 + +# AI Providers & Tools +groq>=0.12.0 +ai-prompter>=0.3 +esperanto>=2.8.3 + +# Content Processing +content-core>=1.0.2 +podcast-creator>=0.7.0 + +# Embeddings & ML +sentence-transformers>=2.2.0 + +# HTTP & Network +httpx[socks]>=0.27.0 +requests>=2.32.0 + +# Study & Learning +fsrs>=6.3.0 + +# Utilities +jinja2>=3.1.0 +tomli>=2.0.2 +packaging>=24.0 + +# Testing (Optional but included for completeness) +pytest>=8.0.0 +pytest-asyncio>=1.2.0 + +surreal-commands>=1.2.0 diff --git a/run_api.py b/run_api.py new file mode 100644 index 0000000000000000000000000000000000000000..17062d26ed56f46b053d6675aa91a3c0537f1022 --- /dev/null +++ b/run_api.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +""" +Startup script for Open Notebook API server. +""" + +import os +import sys +from pathlib import Path + +import uvicorn + +# Add the current directory to Python path so imports work +current_dir = Path(__file__).parent +sys.path.insert(0, str(current_dir)) + +if __name__ == "__main__": + # Default configuration + host = os.getenv("API_HOST", "127.0.0.1") + port = int(os.getenv("API_PORT", "5055")) + reload = os.getenv("API_RELOAD", "true").lower() == "true" + + print(f"Starting Open Notebook API server on {host}:{port}") + print(f"Reload mode: {reload}") + + uvicorn.run( + "api.main:app", + host=host, + port=port, + reload=reload, + reload_dirs=[str(current_dir)] if reload else None, + ) diff --git a/start.sh b/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..92661ce44c3c7107f3cca02e7a9537f448daaa51 --- /dev/null +++ b/start.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +echo "Starting SurrealDB in background..." +surreal start --log debug --user root --pass root memory & +SURREAL_PID=$! + +echo "Waiting 5 seconds for SurrealDB to initialize..." +sleep 5 + +# Check if SurrealDB process is still running +if ! kill -0 $SURREAL_PID 2>/dev/null; then + echo "ERROR: SurrealDB failed to start" + exit 1 +fi + +echo "SurrealDB started successfully (PID: $SURREAL_PID)" +echo "Starting FastAPI application on port 7860..." + +# Set environment variables for Hugging Face deployment +export API_HOST=0.0.0.0 +export API_PORT=7860 +export API_RELOAD=false + +# Start the FastAPI app using run_api.py +exec python run_api.py