Spaces:
Sleeping
Sleeping
ADD: implement cache management, document parsing, file downloading, and Shopify workspace ingestion services
Browse files- services/cache_service.py +28 -0
- services/document_parser_service.py +31 -0
- services/document_service.py +134 -0
- services/file_download_service.py +21 -0
- services/onboarding_service.py +35 -0
- services/product_search_service.py +139 -0
- services/query_service.py +160 -0
- services/shopify_ingest_service.py +82 -0
services/cache_service.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from core.cache import credentials_cache, inventory_cache
|
| 2 |
+
from repositories.social_account_repository import list_channel_identifiers_for_workspace
|
| 3 |
+
from repositories.workspace_repository import get_workspace_id_by_slug
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def clear_all_caches() -> None:
|
| 7 |
+
inventory_cache.clear()
|
| 8 |
+
credentials_cache.clear()
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def clear_query_cache_for_slug(brand_slug: str) -> int:
|
| 12 |
+
workspace_id = get_workspace_id_by_slug(brand_slug)
|
| 13 |
+
channel_identifiers = list_channel_identifiers_for_workspace(workspace_id)
|
| 14 |
+
|
| 15 |
+
return inventory_cache.delete_matching(
|
| 16 |
+
lambda key: any(
|
| 17 |
+
key.startswith(f"query:{identifier}:")
|
| 18 |
+
for identifier in channel_identifiers
|
| 19 |
+
)
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def clear_inventory_cache() -> None:
|
| 24 |
+
inventory_cache.clear()
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def clear_inventory_cache_for_identifier(channel_identifier: str) -> int:
|
| 28 |
+
return inventory_cache.delete_matching(lambda key: channel_identifier in key)
|
services/document_parser_service.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
from io import BytesIO
|
| 3 |
+
|
| 4 |
+
import pdfplumber
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
SUPPORTED_MIME = ["application/pdf", "text/markdown", "text/plain"]
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def parse_file(content: bytes, mime: str) -> str:
|
| 11 |
+
if mime == "application/pdf":
|
| 12 |
+
text = []
|
| 13 |
+
with pdfplumber.open(BytesIO(content)) as pdf:
|
| 14 |
+
for page in pdf.pages:
|
| 15 |
+
page_text = page.extract_text()
|
| 16 |
+
if page_text:
|
| 17 |
+
text.append(page_text.strip())
|
| 18 |
+
return "\n\n".join(text)
|
| 19 |
+
|
| 20 |
+
if mime in ("text/markdown", "text/plain"):
|
| 21 |
+
raw = content.decode("utf-8", errors="ignore")
|
| 22 |
+
if mime == "text/markdown":
|
| 23 |
+
raw = re.sub(r"#{1,6}\s*", "", raw)
|
| 24 |
+
raw = re.sub(r"\*\*(.+?)\*\*", r"\1", raw)
|
| 25 |
+
raw = re.sub(r"\*(.+?)\*", r"\1", raw)
|
| 26 |
+
raw = re.sub(r"`{1,3}.*?`{1,3}", "", raw, flags=re.DOTALL)
|
| 27 |
+
raw = re.sub(r"!\[.*?\]\(.*?\)", "", raw)
|
| 28 |
+
raw = re.sub(r"\[(.+?)\]\(.*?\)", r"\1", raw)
|
| 29 |
+
return raw.strip()
|
| 30 |
+
|
| 31 |
+
raise ValueError(f"Unsupported mime type: {mime}")
|
services/document_service.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
from repositories.document_repository import (
|
| 4 |
+
delete_document_status,
|
| 5 |
+
get_document_status as get_document_status_record,
|
| 6 |
+
upsert_document_status,
|
| 7 |
+
)
|
| 8 |
+
from repositories.embedding_repository import delete_chunks_for_document
|
| 9 |
+
from repositories.social_account_repository import list_channel_identifiers_for_workspace
|
| 10 |
+
from repositories.workspace_repository import get_workspace_id_by_slug
|
| 11 |
+
from rag.chunker import prepare_policy_chunks
|
| 12 |
+
from rag.embedder import delete_chunks_for_source, store_chunks
|
| 13 |
+
from services.cache_service import clear_inventory_cache, clear_inventory_cache_for_identifier
|
| 14 |
+
from services.document_parser_service import SUPPORTED_MIME, parse_file
|
| 15 |
+
from services.file_download_service import download_file
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def validate_supported_mime(mime: str) -> None:
|
| 22 |
+
if mime not in SUPPORTED_MIME:
|
| 23 |
+
raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def resolve_workspace_id(workspace_slug: str) -> str:
|
| 27 |
+
return get_workspace_id_by_slug(workspace_slug)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def ensure_workspace_exists(workspace_slug: str) -> str:
|
| 31 |
+
return resolve_workspace_id(workspace_slug)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def update_status(
|
| 35 |
+
document_id: str,
|
| 36 |
+
workspace_id: str,
|
| 37 |
+
status: str,
|
| 38 |
+
error: str = None,
|
| 39 |
+
chunks: int = 0,
|
| 40 |
+
) -> None:
|
| 41 |
+
upsert_document_status(
|
| 42 |
+
document_id=document_id,
|
| 43 |
+
workspace_id=workspace_id,
|
| 44 |
+
status=status,
|
| 45 |
+
error=error,
|
| 46 |
+
chunks=chunks,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def queue_document_ingest(background_tasks, **kwargs) -> None:
|
| 51 |
+
background_tasks.add_task(process_document, **kwargs)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def process_document(
|
| 55 |
+
workspace_slug: str,
|
| 56 |
+
document_id: str,
|
| 57 |
+
source_url: str,
|
| 58 |
+
mime: str,
|
| 59 |
+
title: str,
|
| 60 |
+
) -> None:
|
| 61 |
+
workspace_id = None
|
| 62 |
+
try:
|
| 63 |
+
validate_supported_mime(mime)
|
| 64 |
+
workspace_id = resolve_workspace_id(workspace_slug)
|
| 65 |
+
update_status(document_id, workspace_id, status="ingesting")
|
| 66 |
+
|
| 67 |
+
logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}")
|
| 68 |
+
|
| 69 |
+
file_bytes = download_file(source_url)
|
| 70 |
+
logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}")
|
| 71 |
+
|
| 72 |
+
text = parse_file(file_bytes, mime)
|
| 73 |
+
if not text.strip():
|
| 74 |
+
raise ValueError("Could not extract any text - file may be empty or image-based")
|
| 75 |
+
|
| 76 |
+
doc = {
|
| 77 |
+
"id": document_id,
|
| 78 |
+
"workspace_id": workspace_id,
|
| 79 |
+
"type": "document",
|
| 80 |
+
"title": title,
|
| 81 |
+
"content": text,
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
delete_chunks_for_source(document_id)
|
| 85 |
+
|
| 86 |
+
chunks = prepare_policy_chunks(doc)
|
| 87 |
+
for chunk in chunks:
|
| 88 |
+
chunk["document_id"] = document_id
|
| 89 |
+
|
| 90 |
+
store_chunks(chunks)
|
| 91 |
+
clear_document_query_cache(workspace_id)
|
| 92 |
+
|
| 93 |
+
update_status(document_id, workspace_id, status="ready", chunks=len(chunks))
|
| 94 |
+
logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}")
|
| 95 |
+
|
| 96 |
+
except Exception as e:
|
| 97 |
+
logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True)
|
| 98 |
+
if workspace_id:
|
| 99 |
+
update_status(document_id, workspace_id, status="failed", error=str(e))
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def clear_document_query_cache(workspace_id: str) -> None:
|
| 103 |
+
cleared = 0
|
| 104 |
+
for identifier in list_channel_identifiers_for_workspace(workspace_id):
|
| 105 |
+
cleared += clear_inventory_cache_for_identifier(identifier)
|
| 106 |
+
|
| 107 |
+
logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}")
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def get_document_status(document_id: str, workspace_slug: str) -> dict:
|
| 111 |
+
workspace_id = resolve_workspace_id(workspace_slug)
|
| 112 |
+
result = get_document_status_record(document_id, workspace_id)
|
| 113 |
+
|
| 114 |
+
if not result:
|
| 115 |
+
raise ValueError("Document not found")
|
| 116 |
+
|
| 117 |
+
return {
|
| 118 |
+
"status": result["status"],
|
| 119 |
+
"error": result.get("error"),
|
| 120 |
+
"chunks": result.get("chunks", 0),
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def delete_document(document_id: str, workspace_slug: str) -> None:
|
| 125 |
+
resolve_workspace_id(workspace_slug)
|
| 126 |
+
|
| 127 |
+
delete_chunks_for_document(document_id)
|
| 128 |
+
delete_document_status(document_id)
|
| 129 |
+
|
| 130 |
+
clear_inventory_cache()
|
| 131 |
+
logger.info(f"Document deleted | id={document_id}")
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
get_workspace_id = resolve_workspace_id
|
services/file_download_service.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import requests
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
MAX_SIZE_MB = 10
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def download_file(source_url: str) -> bytes:
|
| 8 |
+
res = requests.get(source_url, timeout=(5, 60))
|
| 9 |
+
|
| 10 |
+
if res.status_code in (400, 403):
|
| 11 |
+
raise ValueError("Signed URL has expired or is invalid - request a fresh URL and re-ingest")
|
| 12 |
+
if res.status_code == 404:
|
| 13 |
+
raise ValueError("File not found in storage - it may have been deleted")
|
| 14 |
+
|
| 15 |
+
res.raise_for_status()
|
| 16 |
+
|
| 17 |
+
size_mb = len(res.content) / (1024 * 1024)
|
| 18 |
+
if size_mb > MAX_SIZE_MB:
|
| 19 |
+
raise ValueError(f"File too large: {size_mb:.1f}MB - max is {MAX_SIZE_MB}MB")
|
| 20 |
+
|
| 21 |
+
return res.content
|
services/onboarding_service.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
from clients.shopify.admin_api import get_shop_info
|
| 4 |
+
from repositories.social_account_repository import upsert_shopify_account
|
| 5 |
+
from repositories.workspace_repository import upsert_workspace
|
| 6 |
+
from services.shopify_ingest_service import ingest_shopify_workspace
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def domain_to_slug(shop_domain: str) -> str:
|
| 13 |
+
return shop_domain.replace(".myshopify.com", "").strip()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def register_brand(shop_domain: str, access_token: str) -> dict:
|
| 17 |
+
shop = get_shop_info(shop_domain, access_token)
|
| 18 |
+
name = shop["name"]
|
| 19 |
+
slug = domain_to_slug(shop_domain)
|
| 20 |
+
|
| 21 |
+
logger.info(f"Registering brand: {name} ({slug})")
|
| 22 |
+
|
| 23 |
+
workspace = upsert_workspace(name, slug)
|
| 24 |
+
logger.info(f"Workspace registered: {workspace['id']}")
|
| 25 |
+
|
| 26 |
+
upsert_shopify_account(workspace["id"], shop_domain, access_token)
|
| 27 |
+
logger.info(f"Shopify channel linked: {shop_domain}")
|
| 28 |
+
|
| 29 |
+
return workspace
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def onboard_shopify_workspace(shop_domain: str, access_token: str) -> tuple[dict, dict]:
|
| 33 |
+
workspace = register_brand(shop_domain, access_token)
|
| 34 |
+
counts = ingest_shopify_workspace(workspace["slug"])
|
| 35 |
+
return workspace, counts
|
services/product_search_service.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
|
| 4 |
+
from clients.groq import client
|
| 5 |
+
from clients.shopify.product_search_api import search_products
|
| 6 |
+
from rag.prompt_loader import load_prompt
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
REDIRECT_INTENTS = [
|
| 12 |
+
"everything", "all products", "show me all",
|
| 13 |
+
"full catalog", "all items", "everything you have",
|
| 14 |
+
"browse", "what do you have", "show me everything",
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
PRICE_INTENTS = [
|
| 18 |
+
"cheapest", "most expensive", "lowest price",
|
| 19 |
+
"highest price", "most affordable", "budget",
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def is_redirect_query(query: str) -> bool:
|
| 24 |
+
query_lower = query.lower()
|
| 25 |
+
return any(intent in query_lower for intent in REDIRECT_INTENTS)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def is_price_sort_query(query: str) -> bool:
|
| 29 |
+
query_lower = query.lower()
|
| 30 |
+
return any(intent in query_lower for intent in PRICE_INTENTS)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def build_shopify_search_query(user_query: str) -> str:
|
| 34 |
+
response = client.chat.completions.create(
|
| 35 |
+
model="llama-3.1-8b-instant",
|
| 36 |
+
messages=[
|
| 37 |
+
{"role": "system", "content": load_prompt("inventory")},
|
| 38 |
+
{"role": "user", "content": user_query},
|
| 39 |
+
],
|
| 40 |
+
temperature=0,
|
| 41 |
+
)
|
| 42 |
+
return response.choices[0].message.content.strip()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def extract_search_terms(query: str) -> list[str]:
|
| 46 |
+
response = client.chat.completions.create(
|
| 47 |
+
model="llama-3.1-8b-instant",
|
| 48 |
+
messages=[
|
| 49 |
+
{
|
| 50 |
+
"role": "system",
|
| 51 |
+
"content": """Extract product search terms from the user query.
|
| 52 |
+
Return 3 variations: singular, plural, and a related keyword.
|
| 53 |
+
Return ONLY a JSON array of strings, no explanation, no markdown, no backticks."""
|
| 54 |
+
},
|
| 55 |
+
{"role": "user", "content": query},
|
| 56 |
+
],
|
| 57 |
+
temperature=0,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
raw = response.choices[0].message.content.strip()
|
| 61 |
+
try:
|
| 62 |
+
clean = raw.replace("```json", "").replace("```", "").strip()
|
| 63 |
+
return json.loads(clean)
|
| 64 |
+
except Exception:
|
| 65 |
+
return [query]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def format_products_for_llm(products: list[dict]) -> str:
|
| 69 |
+
if not products:
|
| 70 |
+
return "No matching products found."
|
| 71 |
+
|
| 72 |
+
lines = []
|
| 73 |
+
for product in products:
|
| 74 |
+
lines.append(f"\nProduct: {product['title']} ({product['product_type']})")
|
| 75 |
+
if product["description"]:
|
| 76 |
+
lines.append(f" Description: {product['description'][:200]}")
|
| 77 |
+
if product["tags"]:
|
| 78 |
+
lines.append(f" Tags: {', '.join(product['tags'])}")
|
| 79 |
+
for variant in product["variants"]:
|
| 80 |
+
status = "In Stock" if variant["available"] else "Out of Stock"
|
| 81 |
+
lines.append(
|
| 82 |
+
f" - {variant['title']} | Price: ${variant['price']} | "
|
| 83 |
+
f"Stock: {variant['stock']} | {status}"
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
return "\n".join(lines)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def build_inventory_context(shop_domain: str, access_token: str, query: str) -> str:
|
| 90 |
+
if is_redirect_query(query):
|
| 91 |
+
logger.debug("Redirect query detected")
|
| 92 |
+
return f"Please visit our store to browse all products: https://{shop_domain}"
|
| 93 |
+
|
| 94 |
+
if is_price_sort_query(query):
|
| 95 |
+
logger.debug("Price sort query detected")
|
| 96 |
+
products = search_products(shop_domain, access_token, "", limit=250)
|
| 97 |
+
all_variants = []
|
| 98 |
+
for product in products:
|
| 99 |
+
for variant in product["variants"]:
|
| 100 |
+
try:
|
| 101 |
+
all_variants.append({
|
| 102 |
+
"product": product["title"],
|
| 103 |
+
"variant": variant["title"],
|
| 104 |
+
"price": float(variant["price"]),
|
| 105 |
+
"available": variant["available"],
|
| 106 |
+
})
|
| 107 |
+
except Exception:
|
| 108 |
+
continue
|
| 109 |
+
|
| 110 |
+
if not all_variants:
|
| 111 |
+
return "No products found."
|
| 112 |
+
|
| 113 |
+
query_lower = query.lower()
|
| 114 |
+
if any(term in query_lower for term in ["cheapest", "lowest", "affordable", "budget"]):
|
| 115 |
+
result = min(all_variants, key=lambda item: item["price"])
|
| 116 |
+
return f"Our cheapest product is {result['product']} ({result['variant']}) at ${result['price']:.2f} - {'In Stock' if result['available'] else 'Out of Stock'}"
|
| 117 |
+
|
| 118 |
+
result = max(all_variants, key=lambda item: item["price"])
|
| 119 |
+
return f"Our most expensive product is {result['product']} ({result['variant']}) at ${result['price']:.2f} - {'In Stock' if result['available'] else 'Out of Stock'}"
|
| 120 |
+
|
| 121 |
+
shopify_query = build_shopify_search_query(query)
|
| 122 |
+
logger.debug(f"Shopify query built: '{shopify_query}'")
|
| 123 |
+
|
| 124 |
+
products = search_products(shop_domain, access_token, shopify_query)
|
| 125 |
+
logger.debug(f"Found {len(products)} products")
|
| 126 |
+
|
| 127 |
+
if not products:
|
| 128 |
+
fallback_terms = extract_search_terms(query)
|
| 129 |
+
for term in fallback_terms:
|
| 130 |
+
products = search_products(shop_domain, access_token, term)
|
| 131 |
+
if products:
|
| 132 |
+
logger.debug(f"Found {len(products)} with fallback term: '{term}'")
|
| 133 |
+
break
|
| 134 |
+
|
| 135 |
+
return format_products_for_llm(products)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
build_shopify_query = build_shopify_search_query
|
| 139 |
+
handle_product_query = build_inventory_context
|
services/query_service.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
from core.cache import credentials_cache, inventory_cache, rate_limiter
|
| 4 |
+
from repositories.comment_repository import insert_comment
|
| 5 |
+
from repositories.social_account_repository import (
|
| 6 |
+
get_shopify_credentials as get_shopify_credentials_for_workspace,
|
| 7 |
+
get_workspace_by_channel as get_workspace_for_channel,
|
| 8 |
+
)
|
| 9 |
+
from rag.generator import generate_rag_answer, generate_reply_from_contexts
|
| 10 |
+
from rag.query_router import classify_query_route
|
| 11 |
+
from rag.retriever import retrieve
|
| 12 |
+
from services.product_search_service import build_inventory_context
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class RateLimitExceeded(Exception):
|
| 19 |
+
pass
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def get_workspace_by_channel(channel: str, channel_identifier: str) -> tuple:
|
| 23 |
+
try:
|
| 24 |
+
workspace, channel_row = get_workspace_for_channel(channel, channel_identifier)
|
| 25 |
+
except ValueError:
|
| 26 |
+
logger.warning(f"No workspace found for {channel}: {channel_identifier}")
|
| 27 |
+
raise ValueError(f"No workspace found for {channel}: {channel_identifier}")
|
| 28 |
+
|
| 29 |
+
logger.debug(f"Resolved channel {channel_identifier} to workspace {workspace['id']}")
|
| 30 |
+
return workspace, channel_row
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def get_shopify_credentials(workspace_id: str) -> dict:
|
| 34 |
+
cache_key = f"shopify_creds:{workspace_id}"
|
| 35 |
+
cached = credentials_cache.get(cache_key)
|
| 36 |
+
if cached:
|
| 37 |
+
logger.debug(f"Cache hit for Shopify credentials workspace={workspace_id}")
|
| 38 |
+
return cached
|
| 39 |
+
|
| 40 |
+
try:
|
| 41 |
+
credentials = get_shopify_credentials_for_workspace(workspace_id)
|
| 42 |
+
except ValueError:
|
| 43 |
+
logger.warning(f"No Shopify channel found for workspace {workspace_id}")
|
| 44 |
+
raise ValueError(f"No Shopify channel found for workspace {workspace_id}")
|
| 45 |
+
|
| 46 |
+
credentials_cache.set(cache_key, credentials)
|
| 47 |
+
return credentials
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def log_comment(
|
| 51 |
+
workspace_id: str,
|
| 52 |
+
channel: str,
|
| 53 |
+
comment_text: str,
|
| 54 |
+
reply_text: str,
|
| 55 |
+
route: str,
|
| 56 |
+
external_id: str = None,
|
| 57 |
+
commenter_handle: str = None,
|
| 58 |
+
):
|
| 59 |
+
insert_comment(
|
| 60 |
+
workspace_id=workspace_id,
|
| 61 |
+
channel=channel,
|
| 62 |
+
comment_text=comment_text,
|
| 63 |
+
reply_text=reply_text,
|
| 64 |
+
route=route,
|
| 65 |
+
external_id=external_id,
|
| 66 |
+
commenter_handle=commenter_handle,
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def answer(
|
| 71 |
+
query: str,
|
| 72 |
+
channel: str,
|
| 73 |
+
channel_identifier: str,
|
| 74 |
+
external_id: str = None,
|
| 75 |
+
commenter_handle: str = None,
|
| 76 |
+
) -> str:
|
| 77 |
+
logger.info(f"Incoming query | channel={channel} | identifier={channel_identifier}")
|
| 78 |
+
|
| 79 |
+
workspace, _ = get_workspace_by_channel(channel, channel_identifier)
|
| 80 |
+
workspace_id = workspace["id"]
|
| 81 |
+
brand_name = workspace["name"]
|
| 82 |
+
shopify = get_shopify_credentials(workspace_id)
|
| 83 |
+
shop_domain = shopify["channel_identifier"]
|
| 84 |
+
access_token = shopify["credentials"]["access_token"]
|
| 85 |
+
|
| 86 |
+
decision = classify_query_route(query)
|
| 87 |
+
logger.info(f"Router decision: {decision} | query='{query[:60]}'")
|
| 88 |
+
|
| 89 |
+
if decision == "RAG":
|
| 90 |
+
chunks = retrieve(query, workspace_id, top_k=5)
|
| 91 |
+
logger.debug(f"Retrieved {len(chunks)} chunks for RAG")
|
| 92 |
+
reply = generate_rag_answer(query, chunks, brand_name)
|
| 93 |
+
|
| 94 |
+
elif decision == "INVENTORY":
|
| 95 |
+
product_context = build_inventory_context(shop_domain, access_token, query)
|
| 96 |
+
reply = generate_rag_answer(
|
| 97 |
+
query=query,
|
| 98 |
+
chunks=[{"source_type": "inventory", "content": product_context}],
|
| 99 |
+
brand_name=brand_name,
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
elif decision == "BOTH":
|
| 103 |
+
chunks = retrieve(query, workspace_id, top_k=3)
|
| 104 |
+
rag_context = "\n".join([c["content"] for c in chunks])
|
| 105 |
+
product_context = build_inventory_context(shop_domain, access_token, query)
|
| 106 |
+
reply = generate_reply_from_contexts(query, rag_context, product_context, brand_name)
|
| 107 |
+
|
| 108 |
+
else:
|
| 109 |
+
logger.warning(f"Unknown router decision: {decision}")
|
| 110 |
+
reply = "I'm not sure how to answer that."
|
| 111 |
+
decision = "UNKNOWN"
|
| 112 |
+
|
| 113 |
+
log_comment(
|
| 114 |
+
workspace_id=workspace_id,
|
| 115 |
+
channel=channel,
|
| 116 |
+
comment_text=query,
|
| 117 |
+
reply_text=reply,
|
| 118 |
+
route=decision,
|
| 119 |
+
external_id=external_id,
|
| 120 |
+
commenter_handle=commenter_handle,
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
logger.info(f"Reply generated | route={decision} | workspace={workspace_id}")
|
| 124 |
+
return reply
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def answer_query_request(
|
| 128 |
+
query: str,
|
| 129 |
+
channel: str,
|
| 130 |
+
channel_identifier: str,
|
| 131 |
+
external_id: str = None,
|
| 132 |
+
commenter_handle: str = None,
|
| 133 |
+
post_caption: str = None,
|
| 134 |
+
) -> str:
|
| 135 |
+
if not query.strip():
|
| 136 |
+
raise ValueError("Query cannot be empty")
|
| 137 |
+
|
| 138 |
+
if not rate_limiter.is_allowed(channel_identifier):
|
| 139 |
+
logger.warning(f"Rate limit hit for identifier={channel_identifier}")
|
| 140 |
+
raise RateLimitExceeded("Too many requests — please slow down")
|
| 141 |
+
|
| 142 |
+
enriched_query = query
|
| 143 |
+
if post_caption:
|
| 144 |
+
enriched_query = f"Post context: {post_caption}\nUser question: {query}"
|
| 145 |
+
|
| 146 |
+
cache_key = f"query:{channel_identifier}:{query.strip().lower()}"
|
| 147 |
+
cached_reply = inventory_cache.get(cache_key)
|
| 148 |
+
if cached_reply:
|
| 149 |
+
logger.info(f"Cache hit for query='{query[:60]}'")
|
| 150 |
+
return cached_reply
|
| 151 |
+
|
| 152 |
+
reply = answer(
|
| 153 |
+
query=enriched_query,
|
| 154 |
+
channel=channel,
|
| 155 |
+
channel_identifier=channel_identifier,
|
| 156 |
+
external_id=external_id,
|
| 157 |
+
commenter_handle=commenter_handle,
|
| 158 |
+
)
|
| 159 |
+
inventory_cache.set(cache_key, reply)
|
| 160 |
+
return reply
|
services/shopify_ingest_service.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
from clients.shopify.admin_api import get_policies
|
| 4 |
+
from rag.chunker import prepare_policy_chunks
|
| 5 |
+
from rag.embedder import delete_chunks_for_source, store_chunks
|
| 6 |
+
from repositories.social_account_repository import (
|
| 7 |
+
get_shopify_credentials as get_shopify_credentials_for_workspace,
|
| 8 |
+
)
|
| 9 |
+
from repositories.workspace_repository import (
|
| 10 |
+
get_workspace_by_slug,
|
| 11 |
+
list_unindexed_workspaces,
|
| 12 |
+
list_workspace_slugs,
|
| 13 |
+
mark_workspace_indexed,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def get_workspace_by_brand_slug(brand_slug: str) -> dict:
|
| 21 |
+
workspace = get_workspace_by_slug(brand_slug)
|
| 22 |
+
if not workspace:
|
| 23 |
+
raise ValueError(f"No workspace found for slug: {brand_slug}")
|
| 24 |
+
return workspace
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def get_shopify_credentials(workspace_id: str) -> dict:
|
| 28 |
+
return get_shopify_credentials_for_workspace(workspace_id)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def ingest_shopify_workspace(brand_slug: str) -> dict:
|
| 32 |
+
workspace = get_workspace_by_brand_slug(brand_slug)
|
| 33 |
+
workspace_id = workspace["id"]
|
| 34 |
+
channel = get_shopify_credentials(workspace_id)
|
| 35 |
+
shop_domain = channel["channel_identifier"]
|
| 36 |
+
access_token = channel["credentials"]["access_token"]
|
| 37 |
+
|
| 38 |
+
logger.info(f"Ingesting '{workspace['name']}' (slug={brand_slug}) from Shopify")
|
| 39 |
+
counts = {"policies": 0, "faqs": 0}
|
| 40 |
+
|
| 41 |
+
policies = get_policies(shop_domain, access_token, workspace_id)
|
| 42 |
+
for policy in policies:
|
| 43 |
+
delete_chunks_for_source(policy["id"])
|
| 44 |
+
chunks = prepare_policy_chunks(policy)
|
| 45 |
+
store_chunks(chunks)
|
| 46 |
+
counts["policies"] += 1
|
| 47 |
+
logger.info(f"Policy ingested: {policy['title']}")
|
| 48 |
+
|
| 49 |
+
mark_workspace_indexed(workspace_id)
|
| 50 |
+
logger.info(f"Marked workspace '{workspace['name']}' as indexed")
|
| 51 |
+
|
| 52 |
+
return counts
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def sync_new_workspace() -> None:
|
| 56 |
+
unindexed = list_unindexed_workspaces()
|
| 57 |
+
|
| 58 |
+
if not unindexed:
|
| 59 |
+
logger.info("No new workspaces to index")
|
| 60 |
+
return
|
| 61 |
+
|
| 62 |
+
logger.info(f"Found {len(unindexed)} unindexed workspace(s)")
|
| 63 |
+
for brand in unindexed:
|
| 64 |
+
ingest_shopify_workspace(brand["slug"])
|
| 65 |
+
logger.info("Sync complete")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def reindex_brand(brand_slug: str) -> dict:
|
| 69 |
+
logger.info(f"Force re-indexing: {brand_slug}")
|
| 70 |
+
return ingest_shopify_workspace(brand_slug)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def reindex_all() -> None:
|
| 74 |
+
workspaces = list_workspace_slugs()
|
| 75 |
+
logger.info(f"Re-indexing all {len(workspaces)} workspace(s)")
|
| 76 |
+
for brand in workspaces:
|
| 77 |
+
ingest_shopify_workspace(brand["slug"])
|
| 78 |
+
logger.info("Full re-index complete")
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
ingest_brand = ingest_shopify_workspace
|
| 82 |
+
get_brand = get_workspace_by_brand_slug
|