Qurrah commited on
Commit
1c0a11d
·
1 Parent(s): 335bfba

ADD: implement cache management, document parsing, file downloading, and Shopify workspace ingestion services

Browse files
services/cache_service.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from core.cache import credentials_cache, inventory_cache
2
+ from repositories.social_account_repository import list_channel_identifiers_for_workspace
3
+ from repositories.workspace_repository import get_workspace_id_by_slug
4
+
5
+
6
+ def clear_all_caches() -> None:
7
+ inventory_cache.clear()
8
+ credentials_cache.clear()
9
+
10
+
11
+ def clear_query_cache_for_slug(brand_slug: str) -> int:
12
+ workspace_id = get_workspace_id_by_slug(brand_slug)
13
+ channel_identifiers = list_channel_identifiers_for_workspace(workspace_id)
14
+
15
+ return inventory_cache.delete_matching(
16
+ lambda key: any(
17
+ key.startswith(f"query:{identifier}:")
18
+ for identifier in channel_identifiers
19
+ )
20
+ )
21
+
22
+
23
+ def clear_inventory_cache() -> None:
24
+ inventory_cache.clear()
25
+
26
+
27
+ def clear_inventory_cache_for_identifier(channel_identifier: str) -> int:
28
+ return inventory_cache.delete_matching(lambda key: channel_identifier in key)
services/document_parser_service.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from io import BytesIO
3
+
4
+ import pdfplumber
5
+
6
+
7
+ SUPPORTED_MIME = ["application/pdf", "text/markdown", "text/plain"]
8
+
9
+
10
+ def parse_file(content: bytes, mime: str) -> str:
11
+ if mime == "application/pdf":
12
+ text = []
13
+ with pdfplumber.open(BytesIO(content)) as pdf:
14
+ for page in pdf.pages:
15
+ page_text = page.extract_text()
16
+ if page_text:
17
+ text.append(page_text.strip())
18
+ return "\n\n".join(text)
19
+
20
+ if mime in ("text/markdown", "text/plain"):
21
+ raw = content.decode("utf-8", errors="ignore")
22
+ if mime == "text/markdown":
23
+ raw = re.sub(r"#{1,6}\s*", "", raw)
24
+ raw = re.sub(r"\*\*(.+?)\*\*", r"\1", raw)
25
+ raw = re.sub(r"\*(.+?)\*", r"\1", raw)
26
+ raw = re.sub(r"`{1,3}.*?`{1,3}", "", raw, flags=re.DOTALL)
27
+ raw = re.sub(r"!\[.*?\]\(.*?\)", "", raw)
28
+ raw = re.sub(r"\[(.+?)\]\(.*?\)", r"\1", raw)
29
+ return raw.strip()
30
+
31
+ raise ValueError(f"Unsupported mime type: {mime}")
services/document_service.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from repositories.document_repository import (
4
+ delete_document_status,
5
+ get_document_status as get_document_status_record,
6
+ upsert_document_status,
7
+ )
8
+ from repositories.embedding_repository import delete_chunks_for_document
9
+ from repositories.social_account_repository import list_channel_identifiers_for_workspace
10
+ from repositories.workspace_repository import get_workspace_id_by_slug
11
+ from rag.chunker import prepare_policy_chunks
12
+ from rag.embedder import delete_chunks_for_source, store_chunks
13
+ from services.cache_service import clear_inventory_cache, clear_inventory_cache_for_identifier
14
+ from services.document_parser_service import SUPPORTED_MIME, parse_file
15
+ from services.file_download_service import download_file
16
+
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ def validate_supported_mime(mime: str) -> None:
22
+ if mime not in SUPPORTED_MIME:
23
+ raise ValueError(f"Unsupported mime type. Supported: {SUPPORTED_MIME}")
24
+
25
+
26
+ def resolve_workspace_id(workspace_slug: str) -> str:
27
+ return get_workspace_id_by_slug(workspace_slug)
28
+
29
+
30
+ def ensure_workspace_exists(workspace_slug: str) -> str:
31
+ return resolve_workspace_id(workspace_slug)
32
+
33
+
34
+ def update_status(
35
+ document_id: str,
36
+ workspace_id: str,
37
+ status: str,
38
+ error: str = None,
39
+ chunks: int = 0,
40
+ ) -> None:
41
+ upsert_document_status(
42
+ document_id=document_id,
43
+ workspace_id=workspace_id,
44
+ status=status,
45
+ error=error,
46
+ chunks=chunks,
47
+ )
48
+
49
+
50
+ def queue_document_ingest(background_tasks, **kwargs) -> None:
51
+ background_tasks.add_task(process_document, **kwargs)
52
+
53
+
54
+ def process_document(
55
+ workspace_slug: str,
56
+ document_id: str,
57
+ source_url: str,
58
+ mime: str,
59
+ title: str,
60
+ ) -> None:
61
+ workspace_id = None
62
+ try:
63
+ validate_supported_mime(mime)
64
+ workspace_id = resolve_workspace_id(workspace_slug)
65
+ update_status(document_id, workspace_id, status="ingesting")
66
+
67
+ logger.info(f"Processing document | id={document_id} | mime={mime} | workspace={workspace_slug}")
68
+
69
+ file_bytes = download_file(source_url)
70
+ logger.info(f"Downloaded {len(file_bytes) / 1024:.1f}KB for document={document_id}")
71
+
72
+ text = parse_file(file_bytes, mime)
73
+ if not text.strip():
74
+ raise ValueError("Could not extract any text - file may be empty or image-based")
75
+
76
+ doc = {
77
+ "id": document_id,
78
+ "workspace_id": workspace_id,
79
+ "type": "document",
80
+ "title": title,
81
+ "content": text,
82
+ }
83
+
84
+ delete_chunks_for_source(document_id)
85
+
86
+ chunks = prepare_policy_chunks(doc)
87
+ for chunk in chunks:
88
+ chunk["document_id"] = document_id
89
+
90
+ store_chunks(chunks)
91
+ clear_document_query_cache(workspace_id)
92
+
93
+ update_status(document_id, workspace_id, status="ready", chunks=len(chunks))
94
+ logger.info(f"Document embedded | id={document_id} | chunks={len(chunks)}")
95
+
96
+ except Exception as e:
97
+ logger.error(f"Document ingestion failed | id={document_id} | error={e}", exc_info=True)
98
+ if workspace_id:
99
+ update_status(document_id, workspace_id, status="failed", error=str(e))
100
+
101
+
102
+ def clear_document_query_cache(workspace_id: str) -> None:
103
+ cleared = 0
104
+ for identifier in list_channel_identifiers_for_workspace(workspace_id):
105
+ cleared += clear_inventory_cache_for_identifier(identifier)
106
+
107
+ logger.info(f"Cache cleared for workspace={workspace_id} after document ingest | keys={cleared}")
108
+
109
+
110
+ def get_document_status(document_id: str, workspace_slug: str) -> dict:
111
+ workspace_id = resolve_workspace_id(workspace_slug)
112
+ result = get_document_status_record(document_id, workspace_id)
113
+
114
+ if not result:
115
+ raise ValueError("Document not found")
116
+
117
+ return {
118
+ "status": result["status"],
119
+ "error": result.get("error"),
120
+ "chunks": result.get("chunks", 0),
121
+ }
122
+
123
+
124
+ def delete_document(document_id: str, workspace_slug: str) -> None:
125
+ resolve_workspace_id(workspace_slug)
126
+
127
+ delete_chunks_for_document(document_id)
128
+ delete_document_status(document_id)
129
+
130
+ clear_inventory_cache()
131
+ logger.info(f"Document deleted | id={document_id}")
132
+
133
+
134
+ get_workspace_id = resolve_workspace_id
services/file_download_service.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+
3
+
4
+ MAX_SIZE_MB = 10
5
+
6
+
7
+ def download_file(source_url: str) -> bytes:
8
+ res = requests.get(source_url, timeout=(5, 60))
9
+
10
+ if res.status_code in (400, 403):
11
+ raise ValueError("Signed URL has expired or is invalid - request a fresh URL and re-ingest")
12
+ if res.status_code == 404:
13
+ raise ValueError("File not found in storage - it may have been deleted")
14
+
15
+ res.raise_for_status()
16
+
17
+ size_mb = len(res.content) / (1024 * 1024)
18
+ if size_mb > MAX_SIZE_MB:
19
+ raise ValueError(f"File too large: {size_mb:.1f}MB - max is {MAX_SIZE_MB}MB")
20
+
21
+ return res.content
services/onboarding_service.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from clients.shopify.admin_api import get_shop_info
4
+ from repositories.social_account_repository import upsert_shopify_account
5
+ from repositories.workspace_repository import upsert_workspace
6
+ from services.shopify_ingest_service import ingest_shopify_workspace
7
+
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ def domain_to_slug(shop_domain: str) -> str:
13
+ return shop_domain.replace(".myshopify.com", "").strip()
14
+
15
+
16
+ def register_brand(shop_domain: str, access_token: str) -> dict:
17
+ shop = get_shop_info(shop_domain, access_token)
18
+ name = shop["name"]
19
+ slug = domain_to_slug(shop_domain)
20
+
21
+ logger.info(f"Registering brand: {name} ({slug})")
22
+
23
+ workspace = upsert_workspace(name, slug)
24
+ logger.info(f"Workspace registered: {workspace['id']}")
25
+
26
+ upsert_shopify_account(workspace["id"], shop_domain, access_token)
27
+ logger.info(f"Shopify channel linked: {shop_domain}")
28
+
29
+ return workspace
30
+
31
+
32
+ def onboard_shopify_workspace(shop_domain: str, access_token: str) -> tuple[dict, dict]:
33
+ workspace = register_brand(shop_domain, access_token)
34
+ counts = ingest_shopify_workspace(workspace["slug"])
35
+ return workspace, counts
services/product_search_service.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import logging
3
+
4
+ from clients.groq import client
5
+ from clients.shopify.product_search_api import search_products
6
+ from rag.prompt_loader import load_prompt
7
+
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ REDIRECT_INTENTS = [
12
+ "everything", "all products", "show me all",
13
+ "full catalog", "all items", "everything you have",
14
+ "browse", "what do you have", "show me everything",
15
+ ]
16
+
17
+ PRICE_INTENTS = [
18
+ "cheapest", "most expensive", "lowest price",
19
+ "highest price", "most affordable", "budget",
20
+ ]
21
+
22
+
23
+ def is_redirect_query(query: str) -> bool:
24
+ query_lower = query.lower()
25
+ return any(intent in query_lower for intent in REDIRECT_INTENTS)
26
+
27
+
28
+ def is_price_sort_query(query: str) -> bool:
29
+ query_lower = query.lower()
30
+ return any(intent in query_lower for intent in PRICE_INTENTS)
31
+
32
+
33
+ def build_shopify_search_query(user_query: str) -> str:
34
+ response = client.chat.completions.create(
35
+ model="llama-3.1-8b-instant",
36
+ messages=[
37
+ {"role": "system", "content": load_prompt("inventory")},
38
+ {"role": "user", "content": user_query},
39
+ ],
40
+ temperature=0,
41
+ )
42
+ return response.choices[0].message.content.strip()
43
+
44
+
45
+ def extract_search_terms(query: str) -> list[str]:
46
+ response = client.chat.completions.create(
47
+ model="llama-3.1-8b-instant",
48
+ messages=[
49
+ {
50
+ "role": "system",
51
+ "content": """Extract product search terms from the user query.
52
+ Return 3 variations: singular, plural, and a related keyword.
53
+ Return ONLY a JSON array of strings, no explanation, no markdown, no backticks."""
54
+ },
55
+ {"role": "user", "content": query},
56
+ ],
57
+ temperature=0,
58
+ )
59
+
60
+ raw = response.choices[0].message.content.strip()
61
+ try:
62
+ clean = raw.replace("```json", "").replace("```", "").strip()
63
+ return json.loads(clean)
64
+ except Exception:
65
+ return [query]
66
+
67
+
68
+ def format_products_for_llm(products: list[dict]) -> str:
69
+ if not products:
70
+ return "No matching products found."
71
+
72
+ lines = []
73
+ for product in products:
74
+ lines.append(f"\nProduct: {product['title']} ({product['product_type']})")
75
+ if product["description"]:
76
+ lines.append(f" Description: {product['description'][:200]}")
77
+ if product["tags"]:
78
+ lines.append(f" Tags: {', '.join(product['tags'])}")
79
+ for variant in product["variants"]:
80
+ status = "In Stock" if variant["available"] else "Out of Stock"
81
+ lines.append(
82
+ f" - {variant['title']} | Price: ${variant['price']} | "
83
+ f"Stock: {variant['stock']} | {status}"
84
+ )
85
+
86
+ return "\n".join(lines)
87
+
88
+
89
+ def build_inventory_context(shop_domain: str, access_token: str, query: str) -> str:
90
+ if is_redirect_query(query):
91
+ logger.debug("Redirect query detected")
92
+ return f"Please visit our store to browse all products: https://{shop_domain}"
93
+
94
+ if is_price_sort_query(query):
95
+ logger.debug("Price sort query detected")
96
+ products = search_products(shop_domain, access_token, "", limit=250)
97
+ all_variants = []
98
+ for product in products:
99
+ for variant in product["variants"]:
100
+ try:
101
+ all_variants.append({
102
+ "product": product["title"],
103
+ "variant": variant["title"],
104
+ "price": float(variant["price"]),
105
+ "available": variant["available"],
106
+ })
107
+ except Exception:
108
+ continue
109
+
110
+ if not all_variants:
111
+ return "No products found."
112
+
113
+ query_lower = query.lower()
114
+ if any(term in query_lower for term in ["cheapest", "lowest", "affordable", "budget"]):
115
+ result = min(all_variants, key=lambda item: item["price"])
116
+ return f"Our cheapest product is {result['product']} ({result['variant']}) at ${result['price']:.2f} - {'In Stock' if result['available'] else 'Out of Stock'}"
117
+
118
+ result = max(all_variants, key=lambda item: item["price"])
119
+ return f"Our most expensive product is {result['product']} ({result['variant']}) at ${result['price']:.2f} - {'In Stock' if result['available'] else 'Out of Stock'}"
120
+
121
+ shopify_query = build_shopify_search_query(query)
122
+ logger.debug(f"Shopify query built: '{shopify_query}'")
123
+
124
+ products = search_products(shop_domain, access_token, shopify_query)
125
+ logger.debug(f"Found {len(products)} products")
126
+
127
+ if not products:
128
+ fallback_terms = extract_search_terms(query)
129
+ for term in fallback_terms:
130
+ products = search_products(shop_domain, access_token, term)
131
+ if products:
132
+ logger.debug(f"Found {len(products)} with fallback term: '{term}'")
133
+ break
134
+
135
+ return format_products_for_llm(products)
136
+
137
+
138
+ build_shopify_query = build_shopify_search_query
139
+ handle_product_query = build_inventory_context
services/query_service.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from core.cache import credentials_cache, inventory_cache, rate_limiter
4
+ from repositories.comment_repository import insert_comment
5
+ from repositories.social_account_repository import (
6
+ get_shopify_credentials as get_shopify_credentials_for_workspace,
7
+ get_workspace_by_channel as get_workspace_for_channel,
8
+ )
9
+ from rag.generator import generate_rag_answer, generate_reply_from_contexts
10
+ from rag.query_router import classify_query_route
11
+ from rag.retriever import retrieve
12
+ from services.product_search_service import build_inventory_context
13
+
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ class RateLimitExceeded(Exception):
19
+ pass
20
+
21
+
22
+ def get_workspace_by_channel(channel: str, channel_identifier: str) -> tuple:
23
+ try:
24
+ workspace, channel_row = get_workspace_for_channel(channel, channel_identifier)
25
+ except ValueError:
26
+ logger.warning(f"No workspace found for {channel}: {channel_identifier}")
27
+ raise ValueError(f"No workspace found for {channel}: {channel_identifier}")
28
+
29
+ logger.debug(f"Resolved channel {channel_identifier} to workspace {workspace['id']}")
30
+ return workspace, channel_row
31
+
32
+
33
+ def get_shopify_credentials(workspace_id: str) -> dict:
34
+ cache_key = f"shopify_creds:{workspace_id}"
35
+ cached = credentials_cache.get(cache_key)
36
+ if cached:
37
+ logger.debug(f"Cache hit for Shopify credentials workspace={workspace_id}")
38
+ return cached
39
+
40
+ try:
41
+ credentials = get_shopify_credentials_for_workspace(workspace_id)
42
+ except ValueError:
43
+ logger.warning(f"No Shopify channel found for workspace {workspace_id}")
44
+ raise ValueError(f"No Shopify channel found for workspace {workspace_id}")
45
+
46
+ credentials_cache.set(cache_key, credentials)
47
+ return credentials
48
+
49
+
50
+ def log_comment(
51
+ workspace_id: str,
52
+ channel: str,
53
+ comment_text: str,
54
+ reply_text: str,
55
+ route: str,
56
+ external_id: str = None,
57
+ commenter_handle: str = None,
58
+ ):
59
+ insert_comment(
60
+ workspace_id=workspace_id,
61
+ channel=channel,
62
+ comment_text=comment_text,
63
+ reply_text=reply_text,
64
+ route=route,
65
+ external_id=external_id,
66
+ commenter_handle=commenter_handle,
67
+ )
68
+
69
+
70
+ def answer(
71
+ query: str,
72
+ channel: str,
73
+ channel_identifier: str,
74
+ external_id: str = None,
75
+ commenter_handle: str = None,
76
+ ) -> str:
77
+ logger.info(f"Incoming query | channel={channel} | identifier={channel_identifier}")
78
+
79
+ workspace, _ = get_workspace_by_channel(channel, channel_identifier)
80
+ workspace_id = workspace["id"]
81
+ brand_name = workspace["name"]
82
+ shopify = get_shopify_credentials(workspace_id)
83
+ shop_domain = shopify["channel_identifier"]
84
+ access_token = shopify["credentials"]["access_token"]
85
+
86
+ decision = classify_query_route(query)
87
+ logger.info(f"Router decision: {decision} | query='{query[:60]}'")
88
+
89
+ if decision == "RAG":
90
+ chunks = retrieve(query, workspace_id, top_k=5)
91
+ logger.debug(f"Retrieved {len(chunks)} chunks for RAG")
92
+ reply = generate_rag_answer(query, chunks, brand_name)
93
+
94
+ elif decision == "INVENTORY":
95
+ product_context = build_inventory_context(shop_domain, access_token, query)
96
+ reply = generate_rag_answer(
97
+ query=query,
98
+ chunks=[{"source_type": "inventory", "content": product_context}],
99
+ brand_name=brand_name,
100
+ )
101
+
102
+ elif decision == "BOTH":
103
+ chunks = retrieve(query, workspace_id, top_k=3)
104
+ rag_context = "\n".join([c["content"] for c in chunks])
105
+ product_context = build_inventory_context(shop_domain, access_token, query)
106
+ reply = generate_reply_from_contexts(query, rag_context, product_context, brand_name)
107
+
108
+ else:
109
+ logger.warning(f"Unknown router decision: {decision}")
110
+ reply = "I'm not sure how to answer that."
111
+ decision = "UNKNOWN"
112
+
113
+ log_comment(
114
+ workspace_id=workspace_id,
115
+ channel=channel,
116
+ comment_text=query,
117
+ reply_text=reply,
118
+ route=decision,
119
+ external_id=external_id,
120
+ commenter_handle=commenter_handle,
121
+ )
122
+
123
+ logger.info(f"Reply generated | route={decision} | workspace={workspace_id}")
124
+ return reply
125
+
126
+
127
+ def answer_query_request(
128
+ query: str,
129
+ channel: str,
130
+ channel_identifier: str,
131
+ external_id: str = None,
132
+ commenter_handle: str = None,
133
+ post_caption: str = None,
134
+ ) -> str:
135
+ if not query.strip():
136
+ raise ValueError("Query cannot be empty")
137
+
138
+ if not rate_limiter.is_allowed(channel_identifier):
139
+ logger.warning(f"Rate limit hit for identifier={channel_identifier}")
140
+ raise RateLimitExceeded("Too many requests — please slow down")
141
+
142
+ enriched_query = query
143
+ if post_caption:
144
+ enriched_query = f"Post context: {post_caption}\nUser question: {query}"
145
+
146
+ cache_key = f"query:{channel_identifier}:{query.strip().lower()}"
147
+ cached_reply = inventory_cache.get(cache_key)
148
+ if cached_reply:
149
+ logger.info(f"Cache hit for query='{query[:60]}'")
150
+ return cached_reply
151
+
152
+ reply = answer(
153
+ query=enriched_query,
154
+ channel=channel,
155
+ channel_identifier=channel_identifier,
156
+ external_id=external_id,
157
+ commenter_handle=commenter_handle,
158
+ )
159
+ inventory_cache.set(cache_key, reply)
160
+ return reply
services/shopify_ingest_service.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from clients.shopify.admin_api import get_policies
4
+ from rag.chunker import prepare_policy_chunks
5
+ from rag.embedder import delete_chunks_for_source, store_chunks
6
+ from repositories.social_account_repository import (
7
+ get_shopify_credentials as get_shopify_credentials_for_workspace,
8
+ )
9
+ from repositories.workspace_repository import (
10
+ get_workspace_by_slug,
11
+ list_unindexed_workspaces,
12
+ list_workspace_slugs,
13
+ mark_workspace_indexed,
14
+ )
15
+
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ def get_workspace_by_brand_slug(brand_slug: str) -> dict:
21
+ workspace = get_workspace_by_slug(brand_slug)
22
+ if not workspace:
23
+ raise ValueError(f"No workspace found for slug: {brand_slug}")
24
+ return workspace
25
+
26
+
27
+ def get_shopify_credentials(workspace_id: str) -> dict:
28
+ return get_shopify_credentials_for_workspace(workspace_id)
29
+
30
+
31
+ def ingest_shopify_workspace(brand_slug: str) -> dict:
32
+ workspace = get_workspace_by_brand_slug(brand_slug)
33
+ workspace_id = workspace["id"]
34
+ channel = get_shopify_credentials(workspace_id)
35
+ shop_domain = channel["channel_identifier"]
36
+ access_token = channel["credentials"]["access_token"]
37
+
38
+ logger.info(f"Ingesting '{workspace['name']}' (slug={brand_slug}) from Shopify")
39
+ counts = {"policies": 0, "faqs": 0}
40
+
41
+ policies = get_policies(shop_domain, access_token, workspace_id)
42
+ for policy in policies:
43
+ delete_chunks_for_source(policy["id"])
44
+ chunks = prepare_policy_chunks(policy)
45
+ store_chunks(chunks)
46
+ counts["policies"] += 1
47
+ logger.info(f"Policy ingested: {policy['title']}")
48
+
49
+ mark_workspace_indexed(workspace_id)
50
+ logger.info(f"Marked workspace '{workspace['name']}' as indexed")
51
+
52
+ return counts
53
+
54
+
55
+ def sync_new_workspace() -> None:
56
+ unindexed = list_unindexed_workspaces()
57
+
58
+ if not unindexed:
59
+ logger.info("No new workspaces to index")
60
+ return
61
+
62
+ logger.info(f"Found {len(unindexed)} unindexed workspace(s)")
63
+ for brand in unindexed:
64
+ ingest_shopify_workspace(brand["slug"])
65
+ logger.info("Sync complete")
66
+
67
+
68
+ def reindex_brand(brand_slug: str) -> dict:
69
+ logger.info(f"Force re-indexing: {brand_slug}")
70
+ return ingest_shopify_workspace(brand_slug)
71
+
72
+
73
+ def reindex_all() -> None:
74
+ workspaces = list_workspace_slugs()
75
+ logger.info(f"Re-indexing all {len(workspaces)} workspace(s)")
76
+ for brand in workspaces:
77
+ ingest_shopify_workspace(brand["slug"])
78
+ logger.info("Full re-index complete")
79
+
80
+
81
+ ingest_brand = ingest_shopify_workspace
82
+ get_brand = get_workspace_by_brand_slug