Download src/tools/embeddings/encode.py from Snapkitty/sovereign-engine-v2: direct link, hf CLI and curl.
- Browser
- Download file 7.06 kB
-
https://huggingface.co/Snapkitty/sovereign-engine-v2/resolve/main/src/tools/embeddings/encode.py
- Command line
-
hf download hf://Snapkitty/sovereign-engine-v2/src/tools/embeddings/encode.py
-
curl -L -o encode.py https://huggingface.co/Snapkitty/sovereign-engine-v2/resolve/main/src/tools/embeddings/encode.py
7.06 kB
| """ | |
| Embedding Encoding Tools | |
| Part of SOVEREIGN PYTHON LLM ENGINE | |
| Core embedding operations with provider abstraction. | |
| """ | |
| import numpy as np | |
| from typing import Any | |
| import asyncio | |
| from ..registry import tool, RiskClass, ApprovalPolicy | |
| # ========================================== | |
| # Core Tools | |
| # ========================================== | |
| async def encode_text(params: dict[str, Any]) -> dict[str, Any]: | |
| """ | |
| Encode single text to embedding vector. | |
| Args: | |
| params: {text, model, provider} | |
| Returns: | |
| {embedding, dimension, model} | |
| """ | |
| text = params['text'] | |
| model = params.get('model', 'text-embedding-3-small') | |
| provider = params.get('provider', 'openai') | |
| # Get provider implementation | |
| if provider == 'openai': | |
| from .providers.openai import OpenAIEmbeddings | |
| provider_impl = OpenAIEmbeddings() | |
| elif provider == 'cohere': | |
| from .providers.cohere import CohereEmbeddings | |
| provider_impl = CohereEmbeddings() | |
| elif provider == 'local': | |
| from .providers.local import LocalEmbeddings | |
| provider_impl = LocalEmbeddings(model_name=model) | |
| else: | |
| raise ValueError(f"Unknown provider: {provider}") | |
| # Encode | |
| embedding = await provider_impl.encode(text, model=model) | |
| return { | |
| 'embedding': embedding.tolist(), | |
| 'dimension': len(embedding), | |
| 'model': model | |
| } | |
| async def encode_batch(params: dict[str, Any]) -> dict[str, Any]: | |
| """ | |
| Encode multiple texts to embeddings. | |
| Args: | |
| params: {texts, model, provider} | |
| Returns: | |
| {embeddings, dimension, count, model} | |
| """ | |
| texts = params['texts'] | |
| model = params.get('model', 'text-embedding-3-small') | |
| provider = params.get('provider', 'openai') | |
| # Get provider implementation | |
| if provider == 'openai': | |
| from .providers.openai import OpenAIEmbeddings | |
| provider_impl = OpenAIEmbeddings() | |
| elif provider == 'cohere': | |
| from .providers.cohere import CohereEmbeddings | |
| provider_impl = CohereEmbeddings() | |
| elif provider == 'local': | |
| from .providers.local import LocalEmbeddings | |
| provider_impl = LocalEmbeddings(model_name=model) | |
| else: | |
| raise ValueError(f"Unknown provider: {provider}") | |
| # Encode batch | |
| embeddings = await provider_impl.encode_batch(texts, model=model) | |
| return { | |
| 'embeddings': [emb.tolist() for emb in embeddings], | |
| 'dimension': len(embeddings[0]) if embeddings else 0, | |
| 'count': len(embeddings), | |
| 'model': model | |
| } | |
| async def similarity(params: dict[str, Any]) -> dict[str, Any]: | |
| """ | |
| Compute cosine similarity between two vectors. | |
| Args: | |
| params: {embedding1, embedding2} | |
| Returns: | |
| {similarity} | |
| """ | |
| emb1 = np.array(params['embedding1']) | |
| emb2 = np.array(params['embedding2']) | |
| # Cosine similarity | |
| dot_product = np.dot(emb1, emb2) | |
| norm1 = np.linalg.norm(emb1) | |
| norm2 = np.linalg.norm(emb2) | |
| if norm1 == 0 or norm2 == 0: | |
| return {'similarity': 0.0} | |
| sim = float(dot_product / (norm1 * norm2)) | |
| return {'similarity': sim} | |
| async def normalize(params: dict[str, Any]) -> dict[str, Any]: | |
| """ | |
| L2 normalize embedding vector. | |
| Args: | |
| params: {embedding} | |
| Returns: | |
| {normalized} | |
| """ | |
| emb = np.array(params['embedding']) | |
| norm = np.linalg.norm(emb) | |
| if norm == 0: | |
| return {'normalized': emb.tolist()} | |
| normalized = emb / norm | |
| return {'normalized': normalized.tolist()} | |