Buckets:
| """Main Connector class - the primary interface for Context Graph Connector.""" | |
| from __future__ import annotations | |
| import asyncio | |
| from typing import Any | |
| from cgc.adapters.base import DataSource, DiscoveryOptions, FirstN, HealthStatus | |
| from cgc.adapters.graph.base import GraphSink, StorageResult | |
| from cgc.core.chunk import Chunk, ChunkStrategy | |
| from cgc.core.errors import SourceNotFoundError | |
| from cgc.core.graph import RelationshipGraph | |
| from cgc.core.query import Query, QueryResult, SemanticQuery, SqlQuery | |
| from cgc.core.schema import FieldId, Schema | |
| from cgc.core.triplet import Triplet | |
| from cgc.discovery.engine import RelationshipDiscoveryEngine | |
| class SinkNotFoundError(Exception): | |
| """Raised when a graph sink is not found.""" | |
| def __init__(self, sink_id: str): | |
| self.sink_id = sink_id | |
| super().__init__(f"Graph sink not found: {sink_id}") | |
| # NOTE: extract_triplets is imported lazily in the method to avoid loading torch/spacy at startup | |
| class Connector: | |
| """Main interface for Context Graph Connector. | |
| The Connector manages multiple data sources and provides a unified | |
| interface for: | |
| - Schema discovery | |
| - Querying | |
| - Chunking | |
| - Relationship graph construction | |
| - Cross-source operations | |
| Example: | |
| ```python | |
| async with Connector() as cgc: | |
| cgc.add_source(SqlAdapter("db", "postgresql://localhost/app")) | |
| cgc.add_source(FilesystemAdapter("docs", "./documents")) | |
| # Discover schemas | |
| await cgc.discover_all() | |
| # Query data | |
| result = await cgc.sql("db", "SELECT * FROM users LIMIT 10") | |
| # Get relationship graph | |
| graph = await cgc.graph() | |
| ``` | |
| """ | |
| def __init__(self, cache_path: str | None = None): | |
| """Initialize Connector. | |
| Args: | |
| cache_path: Optional path for SQLite cache (not yet implemented) | |
| """ | |
| self._sources: dict[str, DataSource] = {} | |
| self._sinks: dict[str, GraphSink] = {} | |
| self._schemas: dict[str, Schema] = {} | |
| self._graph: RelationshipGraph | None = None | |
| self._discovery_engine = RelationshipDiscoveryEngine() | |
| self._cache_path = cache_path | |
| # === Source Management === | |
| def add_source(self, source: DataSource) -> Connector: | |
| """Add a data source. | |
| Args: | |
| source: DataSource instance to add | |
| Returns: | |
| Self for chaining | |
| """ | |
| self._sources[source.source_id] = source | |
| # Invalidate graph cache | |
| self._graph = None | |
| return self | |
| def remove_source(self, source_id: str) -> bool: | |
| """Remove a data source. | |
| Args: | |
| source_id: ID of source to remove | |
| Returns: | |
| True if source was removed, False if not found | |
| """ | |
| if source_id in self._sources: | |
| del self._sources[source_id] | |
| self._schemas.pop(source_id, None) | |
| self._graph = None | |
| return True | |
| return False | |
| def sources(self) -> list[str]: | |
| """List connected source IDs.""" | |
| return list(self._sources.keys()) | |
| def get_source(self, source_id: str) -> DataSource: | |
| """Get a specific source by ID. | |
| Args: | |
| source_id: Source identifier | |
| Returns: | |
| DataSource instance | |
| Raises: | |
| SourceNotFoundError: If source not found | |
| """ | |
| if source_id not in self._sources: | |
| raise SourceNotFoundError(source_id) | |
| return self._sources[source_id] | |
| def has_source(self, source_id: str) -> bool: | |
| """Check if a source exists.""" | |
| return source_id in self._sources | |
| # === Graph Sink Management === | |
| def add_sink(self, sink: GraphSink) -> Connector: | |
| """Add a graph sink for triplet storage. | |
| Args: | |
| sink: GraphSink instance to add | |
| Returns: | |
| Self for chaining | |
| """ | |
| self._sinks[sink.sink_id] = sink | |
| return self | |
| def remove_sink(self, sink_id: str) -> bool: | |
| """Remove a graph sink. | |
| Args: | |
| sink_id: ID of sink to remove | |
| Returns: | |
| True if sink was removed, False if not found | |
| """ | |
| if sink_id in self._sinks: | |
| del self._sinks[sink_id] | |
| return True | |
| return False | |
| def sinks(self) -> list[str]: | |
| """List connected sink IDs.""" | |
| return list(self._sinks.keys()) | |
| def get_sink(self, sink_id: str) -> GraphSink: | |
| """Get a specific sink by ID. | |
| Args: | |
| sink_id: Sink identifier | |
| Returns: | |
| GraphSink instance | |
| Raises: | |
| SinkNotFoundError: If sink not found | |
| """ | |
| if sink_id not in self._sinks: | |
| raise SinkNotFoundError(sink_id) | |
| return self._sinks[sink_id] | |
| def has_sink(self, sink_id: str) -> bool: | |
| """Check if a sink exists.""" | |
| return sink_id in self._sinks | |
| # === Schema Discovery === | |
| async def discover( | |
| self, | |
| source_id: str, | |
| options: DiscoveryOptions | None = None, | |
| refresh: bool = False, | |
| ) -> Schema: | |
| """Discover schema for a source. | |
| Args: | |
| source_id: Source identifier | |
| options: Discovery options | |
| refresh: Force refresh even if cached | |
| Returns: | |
| Discovered schema | |
| Raises: | |
| asyncio.TimeoutError: If discovery exceeds timeout_seconds | |
| """ | |
| # Check cache | |
| if not refresh and source_id in self._schemas: | |
| return self._schemas[source_id] | |
| options = options or DiscoveryOptions() | |
| # Discover with timeout enforcement | |
| source = self.get_source(source_id) | |
| timeout = options.timeout_seconds | |
| if timeout: | |
| schema = await asyncio.wait_for( | |
| source.discover_schema(options), | |
| timeout=float(timeout), | |
| ) | |
| else: | |
| schema = await source.discover_schema(options) | |
| # Cache | |
| self._schemas[source_id] = schema | |
| return schema | |
| async def discover_all( | |
| self, | |
| options: DiscoveryOptions | None = None, | |
| refresh: bool = False, | |
| ) -> dict[str, Schema]: | |
| """Discover schemas for all sources. | |
| Args: | |
| options: Discovery options (applied to all sources) | |
| refresh: Force refresh | |
| Returns: | |
| Dict mapping source_id to Schema | |
| """ | |
| tasks = [ | |
| self.discover(source_id, options, refresh) | |
| for source_id in self._sources | |
| ] | |
| schemas = await asyncio.gather(*tasks) | |
| return dict(zip(self._sources.keys(), schemas)) | |
| async def schema(self, source_id: str) -> Schema: | |
| """Get schema for a source (discover if needed). | |
| Convenience method that ensures schema is available. | |
| """ | |
| return await self.discover(source_id) | |
| # === Relationship Graph === | |
| async def graph(self, refresh: bool = False) -> RelationshipGraph: | |
| """Get relationship graph across all sources. | |
| Args: | |
| refresh: Force re-discovery | |
| Returns: | |
| RelationshipGraph with all discovered relationships | |
| """ | |
| if not refresh and self._graph: | |
| return self._graph | |
| # Discover all schemas first | |
| schemas = await self.discover_all() | |
| # Run discovery engine | |
| self._graph = self._discovery_engine.discover(list(schemas.values())) | |
| return self._graph | |
| # === Querying === | |
| async def query(self, source_id: str, query: Query) -> QueryResult: | |
| """Execute a query against a source. | |
| Args: | |
| source_id: Source identifier | |
| query: Query object | |
| Returns: | |
| QueryResult | |
| """ | |
| source = self.get_source(source_id) | |
| return await source.query(query) | |
| async def sql(self, source_id: str, sql: str, **params) -> QueryResult: | |
| """Execute a SQL query (convenience method). | |
| Args: | |
| source_id: Source identifier | |
| sql: SQL query string | |
| **params: Query parameters | |
| Returns: | |
| QueryResult | |
| """ | |
| return await self.query(source_id, SqlQuery(sql=sql, params=params)) | |
| async def vector_search( | |
| self, | |
| source_id: str, | |
| collection: str, | |
| query_vector: list[float], | |
| top_k: int = 10, | |
| threshold: float | None = None, | |
| filter: dict[str, Any] | None = None, | |
| ) -> QueryResult: | |
| """Execute a vector similarity search. | |
| Args: | |
| source_id: Source identifier (must be a VectorSource) | |
| collection: Collection/table to search | |
| query_vector: Query embedding vector | |
| top_k: Maximum results | |
| threshold: Minimum similarity score | |
| filter: Metadata filter | |
| Returns: | |
| QueryResult with similarity scores | |
| """ | |
| return await self.query( | |
| source_id, | |
| SemanticQuery( | |
| collection=collection, | |
| query_vector=query_vector, | |
| top_k=top_k, | |
| threshold=threshold, | |
| filter=filter, | |
| ), | |
| ) | |
| # === Chunking === | |
| async def chunk( | |
| self, | |
| source_id: str, | |
| entity: str, | |
| strategy: ChunkStrategy, | |
| ) -> list[Chunk]: | |
| """Chunk data from a source. | |
| Args: | |
| source_id: Source identifier | |
| entity: Entity (table, file) to chunk | |
| strategy: Chunking strategy | |
| Returns: | |
| List of chunks ready for LLM processing | |
| """ | |
| source = self.get_source(source_id) | |
| return await source.chunk(entity, strategy) | |
| # === Sampling === | |
| async def sample( | |
| self, | |
| source_id: str, | |
| entity: str, | |
| n: int = 5, | |
| ) -> list[dict[str, Any]]: | |
| """Sample data from a source. | |
| Args: | |
| source_id: Source identifier | |
| entity: Entity to sample from | |
| n: Number of samples | |
| Returns: | |
| List of sample records | |
| """ | |
| source = self.get_source(source_id) | |
| return await source.sample(entity, FirstN(n)) | |
| # === Cross-Source Operations === | |
| async def find_related( | |
| self, | |
| field_id: FieldId, | |
| value: Any, | |
| ) -> list[dict[str, Any]]: | |
| """Find related records across sources. | |
| Given a field and value, find all related records via | |
| the relationship graph. | |
| Args: | |
| field_id: Starting field | |
| value: Value to search for | |
| Returns: | |
| List of related records with source info | |
| """ | |
| graph = await self.graph() | |
| related = graph.related_to(field_id) | |
| results = [] | |
| for rel in related: | |
| other_field = rel.other_side(field_id) | |
| if not other_field: | |
| continue | |
| # Query the related source | |
| from cgc.core.query import GetQuery | |
| try: | |
| source = self.get_source(other_field.source_id) | |
| result = await source.query( | |
| GetQuery( | |
| entity=other_field.entity, | |
| key=other_field.field, | |
| value=value, | |
| ) | |
| ) | |
| for row in result.to_dicts(): | |
| results.append({ | |
| "source_id": other_field.source_id, | |
| "entity": other_field.entity, | |
| "relationship": rel.relationship_type.value, | |
| "confidence": rel.confidence.value, | |
| "record": row, | |
| }) | |
| except Exception: | |
| continue | |
| return results | |
| # === Triplet Extraction === | |
| def extract_triplets( | |
| self, | |
| text: str, | |
| use_gliner: bool = True, | |
| domain: str | None = None, | |
| ) -> list[Triplet]: | |
| """Extract triplets from text. | |
| Uses hybrid extraction (patterns + GliNER + GliREL) to find | |
| subject-predicate-object relationships with domain-specific | |
| entity and relation labels. | |
| Args: | |
| text: Input text | |
| use_gliner: Whether to use GliNER (higher recall, slower) | |
| domain: Force a specific industry pack (e.g., "tech_startup", | |
| "ecommerce_retail"). None for auto-detection. | |
| Returns: | |
| List of extracted triplets | |
| """ | |
| # Lazy import to avoid loading torch/spacy at startup | |
| from cgc.discovery.extractor import extract_triplets | |
| return extract_triplets(text, use_gliner=use_gliner, domain=domain) | |
| def extract_triplets_structured( | |
| self, | |
| data: list[dict[str, Any]], | |
| ) -> list[Triplet]: | |
| """Extract triplets from structured data using hub-and-spoke model. | |
| Classifies columns into types (primary entity, entity, property, etc.) | |
| and builds relationships between the hub entity and categorical values. | |
| Args: | |
| data: List of row dicts (e.g., from CSV or JSON) | |
| Returns: | |
| List of extracted triplets | |
| """ | |
| from cgc.discovery.structured import StructuredExtractor | |
| extractor = StructuredExtractor() | |
| return extractor.extract_triplets(data) | |
| def extract_file( | |
| self, | |
| path: str, | |
| domain: str | None = None, | |
| use_gliner: bool = True, | |
| ) -> tuple[list[Triplet], str]: | |
| """Extract triplets from a file, auto-detecting structured vs unstructured. | |
| Uses the file parser registry to parse the file, then routes to | |
| structured extraction (hub-and-spoke) if rows are available, or | |
| unstructured extraction (patterns + GliNER) otherwise. | |
| Args: | |
| path: Path to the file | |
| domain: Force a specific industry pack (for unstructured only) | |
| use_gliner: Whether to use GliNER for unstructured extraction | |
| Returns: | |
| Tuple of (triplets, file_type) where file_type is "structured" or "unstructured" | |
| """ | |
| from pathlib import Path as _Path | |
| from cgc.adapters.parsers import parse_file | |
| file_path = _Path(path) | |
| if not file_path.exists(): | |
| raise FileNotFoundError(f"File not found: {path}") | |
| content = file_path.read_bytes() | |
| parsed = parse_file(content, file_path.name) | |
| if parsed.rows: | |
| triplets = self.extract_triplets_structured(parsed.rows) | |
| return triplets, "structured" | |
| else: | |
| triplets = self.extract_triplets(parsed.text, use_gliner=use_gliner, domain=domain) | |
| return triplets, "unstructured" | |
| async def extract_chunked( | |
| self, | |
| source_id: str, | |
| entity: str, | |
| strategy: ChunkStrategy, | |
| use_gliner: bool = True, | |
| domain: str | None = None, | |
| ) -> dict[str, Any]: | |
| """Chunk a file then extract triplets from each chunk. | |
| Designed for unstructured data (PDFs, docs, large text files) where | |
| chunking is needed before extraction. | |
| Args: | |
| source_id: Source containing the file | |
| entity: File entity name | |
| strategy: Chunking strategy to use | |
| use_gliner: Whether to use GliNER for extraction | |
| domain: Force a specific industry pack | |
| Returns: | |
| Dict with per-chunk triplets and aggregate counts | |
| """ | |
| chunks = await self.chunk(source_id, entity, strategy) | |
| results = [] | |
| total_triplets = 0 | |
| for chunk in chunks: | |
| text = chunk.to_text() | |
| triplets = self.extract_triplets(text, use_gliner=use_gliner, domain=domain) | |
| results.append({ | |
| "chunk_index": chunk.index, | |
| "chunk_id": chunk.id, | |
| "triplets": triplets, | |
| "triplet_count": len(triplets), | |
| }) | |
| total_triplets += len(triplets) | |
| return { | |
| "chunks": results, | |
| "total_chunks": len(chunks), | |
| "total_triplets": total_triplets, | |
| } | |
| def detect_domain(self, text: str) -> dict[str, Any]: | |
| """Detect the industry domain of text for optimized extraction. | |
| Routes text to the best-matching industry pack using E5 embeddings. | |
| Args: | |
| text: Input text to classify | |
| Returns: | |
| Dict with pack_id, pack_name, confidence, and all scores | |
| """ | |
| from cgc.discovery.router import create_router | |
| router = create_router() | |
| result = router.route(text) | |
| return { | |
| "pack_id": result.pack.id, | |
| "pack_name": result.pack.name, | |
| "confidence": result.confidence, | |
| "entity_labels": result.pack.entity_labels, | |
| "relation_labels": result.pack.relation_labels, | |
| "scores": result.scores, | |
| } | |
| # === Triplet Storage (Graph Sinks) === | |
| async def store_triplets( | |
| self, | |
| sink_id: str, | |
| triplets: list[Triplet], | |
| graph_name: str | None = None, | |
| merge: bool = True, | |
| ) -> StorageResult: | |
| """Store triplets in a graph sink. | |
| Args: | |
| sink_id: ID of the graph sink to use | |
| triplets: List of triplets to store | |
| graph_name: Optional graph name (for AGE, etc.) | |
| merge: If True, merge with existing nodes/edges | |
| Returns: | |
| StorageResult with counts and any errors | |
| """ | |
| sink = self.get_sink(sink_id) | |
| return await sink.store_triplets(triplets, graph_name, merge) | |
| async def extract_and_store( | |
| self, | |
| sink_id: str, | |
| text: str, | |
| use_gliner: bool = True, | |
| domain: str | None = None, | |
| graph_name: str | None = None, | |
| merge: bool = True, | |
| ) -> dict[str, Any]: | |
| """Extract triplets from text and store in a graph sink. | |
| Combines extraction and storage in one operation. | |
| Args: | |
| sink_id: ID of the graph sink to use | |
| text: Text to extract from | |
| use_gliner: Whether to use GliNER for extraction | |
| domain: Force a specific industry pack | |
| graph_name: Optional graph name (for AGE) | |
| merge: If True, merge with existing nodes/edges | |
| Returns: | |
| Dict with triplets and storage result | |
| """ | |
| triplets = self.extract_triplets(text, use_gliner, domain) | |
| # Store triplets | |
| result = await self.store_triplets(sink_id, triplets, graph_name, merge) | |
| return { | |
| "triplets": [t.to_dict() for t in triplets], | |
| "triplet_count": len(triplets), | |
| "storage_result": { | |
| "nodes_created": result.nodes_created, | |
| "nodes_merged": result.nodes_merged, | |
| "relationships_created": result.relationships_created, | |
| "relationships_merged": result.relationships_merged, | |
| "execution_time_ms": result.execution_time_ms, | |
| "success": result.success, | |
| "errors": result.errors, | |
| }, | |
| } | |
| async def extract_file_and_store( | |
| self, | |
| sink_id: str, | |
| path: str, | |
| domain: str | None = None, | |
| use_gliner: bool = True, | |
| graph_name: str | None = None, | |
| merge: bool = True, | |
| ) -> dict[str, Any]: | |
| """Extract triplets from a file and store in a graph sink. | |
| Args: | |
| sink_id: ID of the graph sink to use | |
| path: Path to file | |
| domain: Force a specific industry pack | |
| use_gliner: Whether to use GliNER | |
| graph_name: Optional graph name (for AGE) | |
| merge: If True, merge with existing nodes/edges | |
| Returns: | |
| Dict with triplets, file type, and storage result | |
| """ | |
| triplets, file_type = self.extract_file(path, domain, use_gliner) | |
| # Store triplets | |
| result = await self.store_triplets(sink_id, triplets, graph_name, merge) | |
| return { | |
| "triplets": [t.to_dict() for t in triplets], | |
| "triplet_count": len(triplets), | |
| "file_type": file_type, | |
| "storage_result": { | |
| "nodes_created": result.nodes_created, | |
| "nodes_merged": result.nodes_merged, | |
| "relationships_created": result.relationships_created, | |
| "relationships_merged": result.relationships_merged, | |
| "execution_time_ms": result.execution_time_ms, | |
| "success": result.success, | |
| "errors": result.errors, | |
| }, | |
| } | |
| async def query_graph_sink( | |
| self, | |
| sink_id: str, | |
| cypher: str, | |
| params: dict[str, Any] | None = None, | |
| graph_name: str | None = None, | |
| ) -> list[dict[str, Any]]: | |
| """Execute a Cypher query against a graph sink. | |
| Args: | |
| sink_id: ID of the graph sink | |
| cypher: Cypher query string | |
| params: Query parameters | |
| graph_name: Optional graph name (for AGE) | |
| Returns: | |
| List of result dictionaries | |
| """ | |
| sink = self.get_sink(sink_id) | |
| return await sink.query_graph(cypher, params, graph_name) | |
| async def find_entity_in_sink( | |
| self, | |
| sink_id: str, | |
| entity: str, | |
| graph_name: str | None = None, | |
| limit: int = 100, | |
| ) -> list[dict[str, Any]]: | |
| """Find all triplets involving an entity in a graph sink. | |
| Args: | |
| sink_id: ID of the graph sink | |
| entity: Entity name to search for | |
| graph_name: Optional graph name (for AGE) | |
| limit: Maximum results | |
| Returns: | |
| List of triplet dictionaries | |
| """ | |
| sink = self.get_sink(sink_id) | |
| return await sink.find_by_entity(entity, graph_name, limit) | |
| # === Health Checks === | |
| async def health_check(self) -> dict[str, HealthStatus]: | |
| """Check health of all sources. | |
| Returns: | |
| Dict mapping source_id to HealthStatus | |
| """ | |
| tasks = [source.health_check() for source in self._sources.values()] | |
| results = await asyncio.gather(*tasks, return_exceptions=True) | |
| health = {} | |
| for source_id, result in zip(self._sources.keys(), results): | |
| if isinstance(result, Exception): | |
| health[source_id] = HealthStatus( | |
| healthy=False, | |
| message=str(result), | |
| ) | |
| else: | |
| health[source_id] = result | |
| return health | |
| async def health_check_source(self, source_id: str) -> HealthStatus: | |
| """Check health of a specific source.""" | |
| source = self.get_source(source_id) | |
| return await source.health_check() | |
| # === Utilities === | |
| def summary(self) -> str: | |
| """Generate compact summary for LLM context. | |
| Returns summary of all connected sources and their schemas. | |
| """ | |
| lines = [f"Connected sources: {len(self._sources)}"] | |
| for source_id, schema in self._schemas.items(): | |
| lines.append(f"\n{schema.to_compact()}") | |
| if self._graph: | |
| lines.append(f"\nRelationships: {len(self._graph.relationships)}") | |
| return "\n".join(lines) | |
| def to_dict(self) -> dict[str, Any]: | |
| """Convert connector state to dictionary.""" | |
| return { | |
| "sources": list(self._sources.keys()), | |
| "schemas": { | |
| sid: schema.to_dict() for sid, schema in self._schemas.items() | |
| }, | |
| "relationships": ( | |
| self._graph.to_dict() if self._graph else {"relationships": [], "total": 0} | |
| ), | |
| } | |
| async def close(self) -> None: | |
| """Close all connections.""" | |
| for source in self._sources.values(): | |
| await source.close() | |
| for sink in self._sinks.values(): | |
| await sink.close() | |
| async def __aenter__(self) -> Connector: | |
| return self | |
| async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: | |
| await self.close() | |
| class ConnectorBuilder: | |
| """Builder for creating Connector instances. | |
| Provides a fluent interface for adding sources and graph sinks. | |
| Example: | |
| ```python | |
| connector = ( | |
| ConnectorBuilder() | |
| .add_postgres("db", "postgresql://localhost/app") | |
| .add_filesystem("docs", "./documents") | |
| .add_qdrant("vectors", "http://localhost:6333") | |
| .add_neo4j("graph", "bolt://localhost:7687", "neo4j", "password") | |
| .build() | |
| ) | |
| ``` | |
| """ | |
| def __init__(self): | |
| self._sources: list[DataSource] = [] | |
| self._sinks: list[GraphSink] = [] | |
| self._cache_path: str | None = None | |
| def add_source(self, source: DataSource) -> ConnectorBuilder: | |
| """Add a generic data source.""" | |
| self._sources.append(source) | |
| return self | |
| def add_postgres( | |
| self, | |
| source_id: str, | |
| connection_string: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a PostgreSQL source.""" | |
| from cgc.adapters.sql import SqlAdapter | |
| self._sources.append(SqlAdapter(source_id, connection_string, **kwargs)) | |
| return self | |
| def add_mysql( | |
| self, | |
| source_id: str, | |
| connection_string: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a MySQL source.""" | |
| from cgc.adapters.sql import SqlAdapter | |
| self._sources.append(SqlAdapter(source_id, connection_string, **kwargs)) | |
| return self | |
| def add_sqlite( | |
| self, | |
| source_id: str, | |
| path: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a SQLite source.""" | |
| from cgc.adapters.sql import SqlAdapter | |
| self._sources.append(SqlAdapter(source_id, f"sqlite:///{path}", **kwargs)) | |
| return self | |
| def add_filesystem( | |
| self, | |
| source_id: str, | |
| path: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a local filesystem source.""" | |
| from cgc.adapters.filesystem import FilesystemAdapter | |
| self._sources.append(FilesystemAdapter(source_id, path, **kwargs)) | |
| return self | |
| def add_pgvector( | |
| self, | |
| source_id: str, | |
| connection_string: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a pgvector source.""" | |
| from cgc.adapters.vector.pgvector import PgVectorAdapter | |
| self._sources.append(PgVectorAdapter(source_id, connection_string, **kwargs)) | |
| return self | |
| def add_qdrant( | |
| self, | |
| source_id: str, | |
| url: str = "http://localhost:6333", | |
| api_key: str | None = None, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a Qdrant source.""" | |
| from cgc.adapters.vector.qdrant import QdrantAdapter | |
| self._sources.append(QdrantAdapter(source_id, url, api_key, **kwargs)) | |
| return self | |
| def add_pinecone( | |
| self, | |
| source_id: str, | |
| api_key: str, | |
| index_name: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a Pinecone source.""" | |
| from cgc.adapters.vector.pinecone import PineconeAdapter | |
| self._sources.append(PineconeAdapter(source_id, api_key, index_name, **kwargs)) | |
| return self | |
| def add_mongodb_vector( | |
| self, | |
| source_id: str, | |
| connection_string: str, | |
| database: str, | |
| **kwargs, | |
| ) -> ConnectorBuilder: | |
| """Add a MongoDB Atlas Vector Search source.""" | |
| from cgc.adapters.vector.mongodb import MongoVectorAdapter | |
| self._sources.append( | |
| MongoVectorAdapter(source_id, connection_string, database, **kwargs) | |
| ) | |
| return self | |
| def with_cache(self, path: str) -> ConnectorBuilder: | |
| """Set cache path for schema caching.""" | |
| self._cache_path = path | |
| return self | |
| # === Graph Sinks === | |
| def add_sink(self, sink: GraphSink) -> ConnectorBuilder: | |
| """Add a generic graph sink.""" | |
| self._sinks.append(sink) | |
| return self | |
| def add_neo4j( | |
| self, | |
| sink_id: str, | |
| uri: str, | |
| user: str | None = None, | |
| password: str | None = None, | |
| database: str | None = None, | |
| ) -> ConnectorBuilder: | |
| """Add a Neo4j graph sink. | |
| Args: | |
| sink_id: Unique identifier for this sink | |
| uri: Neo4j connection URI (bolt://host:port) | |
| user: Username for authentication | |
| password: Password for authentication | |
| database: Database name (default: neo4j) | |
| """ | |
| from cgc.adapters.graph.neo4j import Neo4jAdapter | |
| self._sinks.append(Neo4jAdapter(sink_id, uri, user, password, database)) | |
| return self | |
| def add_age( | |
| self, | |
| sink_id: str, | |
| connection: str, | |
| graph_name: str = "cgc_graph", | |
| ) -> ConnectorBuilder: | |
| """Add a PostgreSQL Apache AGE graph sink. | |
| Args: | |
| sink_id: Unique identifier for this sink | |
| connection: PostgreSQL connection string | |
| graph_name: Name of the graph to use/create | |
| """ | |
| from cgc.adapters.graph.age import AgeAdapter | |
| self._sinks.append(AgeAdapter(sink_id, connection, graph_name)) | |
| return self | |
| def build(self) -> Connector: | |
| """Build the Connector instance.""" | |
| connector = Connector(cache_path=self._cache_path) | |
| for source in self._sources: | |
| connector.add_source(source) | |
| for sink in self._sinks: | |
| connector.add_sink(sink) | |
| return connector | |
Xet Storage Details
- Size:
- 29.9 kB
- Xet hash:
- 28366b8c01d668c9d4789d2c8067556c7d6242c2b99a5b1db4310154741eea0f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.