Download backend/core/gemini_agent.py from StillWorking/sentinel-face-recognition: direct link, hf CLI and curl.
- Browser
- Download file 29.1 kB
-
https://huggingface.co/StillWorking/sentinel-face-recognition/resolve/main/backend/core/gemini_agent.py
- Command line
-
hf download hf://StillWorking/sentinel-face-recognition/backend/core/gemini_agent.py
-
curl -L -o gemini_agent.py https://huggingface.co/StillWorking/sentinel-face-recognition/resolve/main/backend/core/gemini_agent.py
29.1 kB
| """ | |
| core/gemini_agent.py β Sentinel-AI | |
| ====================================== | |
| Wraps the Google GenAI SDK with cascadeflow guardrails to provide two | |
| async functions: | |
| β’ generate_alert(attributes, similarity) | |
| Drafts a professional security alert from matched face attributes. | |
| β’ extract_attributes_from_query(query) | |
| Parses a natural-language operator query into a list of CelebA | |
| attribute names that can be used to filter the FAISS metadata. | |
| Both functions are decorated with @cascadeflow.agent (policy metadata) and | |
| execute inside a cascadeflow.run() scope to record latency and cost budget. | |
| Every cloud provider is optional at runtime: Gemini β Groq β Cohere β | |
| deterministic local fallback. Importing this module never requires an API | |
| key; clients are created lazily on first use. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import time | |
| from typing import Any | |
| import cascadeflow | |
| from core.paths import env_float | |
| # cascadeflow.agent at the top level resolves to the agent.py module (not a | |
| # callable) due to lazy-import aliasing. The correct decorator is | |
| # cascadeflow.harness_agent, which is the harness.api.agent function. | |
| _cf_agent = cascadeflow.harness_agent | |
| _cf_run = cascadeflow.run | |
| import cohere | |
| from google import genai | |
| from google.genai import types as genai_types | |
| from groq import AsyncGroq | |
| logger = logging.getLogger(__name__) | |
| # Load fallback models from .env | |
| GROQ_MODEL = os.getenv("GROQ_MODEL", "openai/gpt-oss-120b") | |
| COHERE_MODEL = os.getenv("COHERE_MODEL", "command-r-plus-08-2024") | |
| # Instantiate fallback clients lazily | |
| _groq_client: AsyncGroq | None = None | |
| _cohere_client: cohere.AsyncClientV2 | None = None | |
| def _get_groq_client() -> AsyncGroq: | |
| global _groq_client | |
| if _groq_client is None: | |
| api_key = os.getenv("GROQ_API_KEY") | |
| if not api_key: | |
| raise RuntimeError("GROQ_API_KEY is not configured.") | |
| _groq_client = AsyncGroq(api_key=api_key) | |
| return _groq_client | |
| def _get_cohere_client() -> cohere.AsyncClientV2: | |
| global _cohere_client | |
| if _cohere_client is None: | |
| api_key = os.getenv("COHERE_API_KEY") | |
| if not api_key: | |
| raise RuntimeError("COHERE_API_KEY is not configured.") | |
| # AsyncClientV2 is required: the sync client would block the event loop. | |
| _cohere_client = cohere.AsyncClientV2(api_key=api_key) | |
| return _cohere_client | |
| def warm_providers() -> list[str]: | |
| """ | |
| Initialise every configured provider client at startup. | |
| Client construction is synchronous (auth discovery, HTTP pool setup) and | |
| otherwise lands on the first live request, where it can add ~1 s and push | |
| an otherwise in-budget call over the deadline. Missing keys are skipped. | |
| """ | |
| warmed: list[str] = [] | |
| for name, factory in ( | |
| ("Gemini", _get_client), | |
| ("Groq", _get_groq_client), | |
| ("Cohere", _get_cohere_client), | |
| ): | |
| try: | |
| factory() | |
| warmed.append(name) | |
| except Exception as exc: | |
| logger.info("%s client not warmed: %s", name, exc) | |
| return warmed | |
| # ββ Configuration βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Accept both GEMINI_API_KEY (per spec) and GOOGLE_API_KEY (legacy) so either | |
| # variable in .env works without breaking existing setups. | |
| GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.5-flash") | |
| # Per-call guardrail budgets (can be overridden via env) | |
| _ALERT_BUDGET_USD = float(os.getenv("CASCADEFLOW_ALERT_BUDGET", "0.01")) | |
| _SEARCH_BUDGET_USD = float(os.getenv("CASCADEFLOW_SEARCH_BUDGET", "0.005")) | |
| _LATENCY_LIMIT_MS = float(os.getenv("CASCADEFLOW_LATENCY_LIMIT_MS", "2000")) | |
| # cascadeflow records ``max_latency_ms`` but does not cancel a slow provider. | |
| # These two values are enforced with ``asyncio.wait_for``: | |
| # β’ LLM_HARD_TIMEOUT_MS β ceiling for the whole GeminiβGroqβCohere chain | |
| # β’ LLM_PROVIDER_TIMEOUT_MS β ceiling for a single provider attempt | |
| # A slow Gemini call is cancelled at the provider deadline, leaving the rest of | |
| # the budget for a fallback. Raise LLM_HARD_TIMEOUT_MS if you prefer answer | |
| # quality over the 2 s target. | |
| _LLM_HARD_TIMEOUT_MS = max( | |
| 100.0, | |
| env_float("LLM_HARD_TIMEOUT_MS", _LATENCY_LIMIT_MS), | |
| ) | |
| _LLM_PROVIDER_TIMEOUT_MS = max( | |
| 50.0, | |
| env_float("LLM_PROVIDER_TIMEOUT_MS", max(250.0, _LATENCY_LIMIT_MS * 0.6)), | |
| ) | |
| # ββ Lazy Gemini client singleton ββββββββββββββββββββββββββββββββββββββββββββββ | |
| _client: genai.Client | None = None | |
| def _get_client() -> genai.Client: | |
| """Return the Gemini client, initialising it on first call.""" | |
| global _client | |
| if _client is None: | |
| api_key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") or "" | |
| if not api_key: | |
| raise RuntimeError( | |
| "No Gemini API key found. " | |
| "Set GEMINI_API_KEY in your .env file " | |
| "(copy .env.example β .env and fill in the value)." | |
| ) | |
| _client = genai.Client(api_key=api_key) | |
| return _client | |
| def _safe_text(value: Any) -> str: | |
| """Return ``value.strip()`` for strings, otherwise an empty string.""" | |
| return value.strip() if isinstance(value, str) else "" | |
| def _cohere_text(response: Any) -> str: | |
| """Extract the first text block from a Cohere V2 chat response.""" | |
| message = getattr(response, "message", None) | |
| content = getattr(message, "content", None) | |
| if not content: | |
| return "" | |
| return _safe_text(getattr(content[0], "text", None)) | |
| async def _await_with_deadline(coro, deadline: float): | |
| """ | |
| Await ``coro`` with the per-provider and whole-chain deadlines applied. | |
| ``deadline`` is a ``time.monotonic()`` value. ``asyncio.wait_for`` waits | |
| for the cancelled provider to finish cleaning up, which can itself take | |
| hundreds of milliseconds (the google-genai client closes an HTTP stream). | |
| That overran the wall-clock budget in testing, so this helper races the | |
| task against a timer instead: on expiry it cancels the task, attaches a | |
| callback to consume the eventual result, and raises immediately. | |
| """ | |
| remaining = deadline - time.monotonic() | |
| if remaining <= 0: | |
| coro.close() | |
| raise asyncio.TimeoutError("LLM hard deadline exceeded.") | |
| provider_timeout = min(remaining, _LLM_PROVIDER_TIMEOUT_MS / 1000.0) | |
| task = asyncio.ensure_future(coro) | |
| done, _pending = await asyncio.wait({task}, timeout=provider_timeout) | |
| if task in done: | |
| return task.result() | |
| task.cancel() | |
| def _consume_result(finished: asyncio.Future) -> None: | |
| # Retrieve the exception so asyncio does not log "exception was never | |
| # retrieved" when the provider finishes cancelling in the background. | |
| if not finished.cancelled(): | |
| try: | |
| finished.exception() | |
| except BaseException: | |
| pass | |
| task.add_done_callback(_consume_result) | |
| raise asyncio.TimeoutError("LLM hard deadline exceeded.") | |
| # ββ Valid CelebA attribute names (used in the extraction prompt) ββββββββββββββ | |
| CELEBA_ATTRIBUTES = [ | |
| "5_o_Clock_Shadow", "Arched_Eyebrows", "Attractive", "Bags_Under_Eyes", | |
| "Bald", "Bangs", "Big_Lips", "Big_Nose", "Black_Hair", "Blond_Hair", | |
| "Blurry", "Brown_Hair", "Bushy_Eyebrows", "Chubby", "Double_Chin", | |
| "Eyeglasses", "Goatee", "Gray_Hair", "Heavy_Makeup", "High_Cheekbones", | |
| "Male", "Mouth_Slightly_Open", "Mustache", "Narrow_Eyes", "No_Beard", | |
| "Oval_Face", "Pale_Skin", "Pointy_Nose", "Receding_Hairline", | |
| "Rosy_Cheeks", "Sideburns", "Smiling", "Straight_Hair", "Wavy_Hair", | |
| "Wearing_Earrings", "Wearing_Hat", "Wearing_Lipstick", "Wearing_Necklace", | |
| "Wearing_Necktie", "Young", | |
| ] | |
| _ATTR_LIST_STR = ", ".join(CELEBA_ATTRIBUTES) | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Local keyword fallback | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # Ordered longest-phrase-first so "black hair" wins over a bare "hair" rule. | |
| # Generic "beard" is intentionally absent: CelebA has no positive "Beard" | |
| # attribute and mapping it to the inverse "No_Beard" returned beardless | |
| # subjects. "No beard" is handled explicitly below. | |
| _KEYWORD_MAP: tuple[tuple[str, str], ...] = ( | |
| ("5 o'clock shadow", "5_o_Clock_Shadow"), | |
| ("five o'clock shadow", "5_o_Clock_Shadow"), | |
| ("black hair", "Black_Hair"), | |
| ("blond hair", "Blond_Hair"), | |
| ("blonde hair", "Blond_Hair"), | |
| ("brown hair", "Brown_Hair"), | |
| ("gray hair", "Gray_Hair"), | |
| ("grey hair", "Gray_Hair"), | |
| ("straight hair", "Straight_Hair"), | |
| ("rosy cheeks", "Rosy_Cheeks"), | |
| ("receding hairline", "Receding_Hairline"), | |
| ("bushy eyebrows", "Bushy_Eyebrows"), | |
| ("arched eyebrows", "Arched_Eyebrows"), | |
| ("bags under eyes", "Bags_Under_Eyes"), | |
| ("bags under the eyes", "Bags_Under_Eyes"), | |
| ("double chin", "Double_Chin"), | |
| ("big lips", "Big_Lips"), | |
| ("big nose", "Big_Nose"), | |
| ("pointy nose", "Pointy_Nose"), | |
| ("high cheekbones", "High_Cheekbones"), | |
| ("heavy makeup", "Heavy_Makeup"), | |
| ("mouth slightly open", "Mouth_Slightly_Open"), | |
| ("pale skin", "Pale_Skin"), | |
| ("wearing a hat", "Wearing_Hat"), | |
| ("hat", "Wearing_Hat"), | |
| ("wearing a necklace", "Wearing_Necklace"), | |
| ("wearing a necktie", "Wearing_Necktie"), | |
| ("wearing a tie", "Wearing_Necktie"), | |
| ("earrings", "Wearing_Earrings"), | |
| ("necklace", "Wearing_Necklace"), | |
| ("necktie", "Wearing_Necktie"), | |
| ("eyeglasses", "Eyeglasses"), | |
| ("glasses", "Eyeglasses"), | |
| ("spectacles", "Eyeglasses"), | |
| ("blurry", "Blurry"), | |
| ("blurred", "Blurry"), | |
| ("mustache", "Mustache"), | |
| ("moustache", "Mustache"), | |
| ("goatee", "Goatee"), | |
| ("sideburns", "Sideburns"), | |
| ("stubble", "5_o_Clock_Shadow"), | |
| ("bald", "Bald"), | |
| ("bangs", "Bangs"), | |
| ("fringe", "Bangs"), | |
| ("wavy", "Wavy_Hair"), | |
| ("curly", "Wavy_Hair"), | |
| ("straight hair", "Straight_Hair"), | |
| ("lipstick", "Wearing_Lipstick"), | |
| ("makeup", "Heavy_Makeup"), | |
| ("smiling", "Smiling"), | |
| ("smile", "Smiling"), | |
| ("chubby", "Chubby"), | |
| ("attractive", "Attractive"), | |
| ("pale", "Pale_Skin"), | |
| ) | |
| _FEMALE_WORDS = ("woman", "women", "female", "girl", "lady", "she", "her", "hers") | |
| _MALE_WORDS = ("man", "men", "male", "boy", "guy", "gentleman", "he", "him", "his") | |
| _YOUNG_WORDS = ("young", "youth", "teen", "teenager", "teenage", "child", "kid") | |
| _OLD_WORDS = ("old", "older", "elderly", "senior", "aged", "middle-aged") | |
| def _has_words(text: str, words: tuple[str, ...]) -> bool: | |
| """Word-boundary match to avoid 'he' inside 'there' / 'her' inside 'other'.""" | |
| pattern = r"\b(?:" + "|".join(re.escape(w) for w in words) + r")\b" | |
| return re.search(pattern, text, flags=re.IGNORECASE) is not None | |
| # Looks backwards from a keyword for a nearby negation: | |
| # "no glasses", "without a hat", "not wearing makeup", | |
| # "doesn't have a beard", "free of earrings". | |
| _NEGATION_RE = re.compile( | |
| r"\b(?:no|without|not(?:\s+(?:wearing|have|having))?|" | |
| r"isn't|aren't|doesn't(?:\s+have)?|don't|dont|zero|free\s+of)" | |
| r"\s+(?:\w+\s+){0,3}$", | |
| re.IGNORECASE, | |
| ) | |
| def _words_with_negation(text: str, words: tuple[str, ...]) -> tuple[bool, bool]: | |
| """ | |
| Return ``(positive, negated)`` for a set of words. | |
| Each occurrence is classified independently, so "a woman, not a man" | |
| counts as positive-female and negated-male. | |
| """ | |
| pattern = r"\b(?:" + "|".join(re.escape(w) for w in words) + r")\b" | |
| positive = negated = False | |
| for match in re.finditer(pattern, text, flags=re.IGNORECASE): | |
| if _NEGATION_RE.search(text[: match.start()]): | |
| negated = True | |
| else: | |
| positive = True | |
| return positive, negated | |
| def _keyword_filter(query: str) -> tuple[list[str], list[str]]: | |
| """ | |
| Deterministic offline extraction of required *and excluded* attributes. | |
| CelebA encodes gender and age as single bits (``Male``, ``Young``); there | |
| is no ``Female`` or ``Old`` column. A query for "woman" therefore cannot | |
| be expressed as a positive filter alone β returning every face that is | |
| smiling and blond includes men. Negative constraints close that gap. | |
| Negated keywords ("no glasses", "without a hat") are likewise moved to | |
| the exclusion list instead of being added as positive requirements, which | |
| is the same class of bug as mapping "beard" to ``No_Beard``. | |
| """ | |
| query_lower = query.lower() | |
| attrs: list[str] = [] | |
| excluded: list[str] = [] | |
| def add(bucket: list[str], attr: str) -> None: | |
| if attr not in bucket: | |
| bucket.append(attr) | |
| for keyword, attr in _KEYWORD_MAP: | |
| for match in re.finditer(rf"\b{re.escape(keyword)}\b", query_lower): | |
| if _NEGATION_RE.search(query_lower[: match.start()]): | |
| add(excluded, attr) | |
| else: | |
| add(attrs, attr) | |
| male_pos, male_neg = _words_with_negation(query_lower, _MALE_WORDS) | |
| female_pos, female_neg = _words_with_negation(query_lower, _FEMALE_WORDS) | |
| if female_pos and not male_pos: | |
| add(excluded, "Male") | |
| elif male_pos and not female_pos: | |
| add(attrs, "Male") | |
| elif male_neg and not female_neg: | |
| # "not a man" / "no men" β non-male is the intended constraint. | |
| add(excluded, "Male") | |
| young_pos, young_neg = _words_with_negation(query_lower, _YOUNG_WORDS) | |
| old_pos, old_neg = _words_with_negation(query_lower, _OLD_WORDS) | |
| if old_pos and not young_pos: | |
| add(excluded, "Young") | |
| elif young_pos and not old_pos: | |
| add(attrs, "Young") | |
| elif young_neg and not old_neg: | |
| add(excluded, "Young") | |
| # "no beard"/"without a beard"/"beardless" is the only correct way to use | |
| # No_Beard (CelebA has no positive "Beard" label). | |
| if re.search(r"\b(?:no|without(?:\s+a)?)\s+beard\b|\bbeardless\b", query_lower): | |
| add(attrs, "No_Beard") | |
| # "clean-shaven" implies all three facial-hair labels are absent. | |
| if re.search(r"\bclean[- ]shaven\b", query_lower): | |
| add(attrs, "No_Beard") | |
| for attr in ("Mustache", "Goatee", "5_o_Clock_Shadow"): | |
| add(excluded, attr) | |
| # An attribute cannot be both required and forbidden. | |
| excluded_set = {a.lower() for a in excluded} | |
| attrs = [a for a in attrs if a.lower() not in excluded_set] | |
| return ( | |
| [a for a in attrs if a in CELEBA_ATTRIBUTES], | |
| [a for a in excluded if a in CELEBA_ATTRIBUTES], | |
| ) | |
| def _keyword_extract(query: str) -> list[str]: | |
| """Backward-compatible positive-only view of :func:`_keyword_filter`.""" | |
| return _keyword_filter(query)[0] | |
| def _strip_code_fences(text: str) -> str: | |
| """Remove ```json β¦ ``` wrappers that models sometimes emit.""" | |
| text = text.strip() | |
| if text.startswith("```"): | |
| text = re.sub(r"^```[a-zA-Z0-9_-]*\s*", "", text) | |
| text = re.sub(r"\s*```$", "", text) | |
| return text.strip() | |
| def _parse_attributes(raw_response: str) -> list[str]: | |
| """ | |
| Parse an LLM response into validated CelebA attribute names. | |
| Accepts a bare JSON array, ``{"attributes": [...]}``, any object | |
| containing an array, or a JSON array embedded in prose/code fences. | |
| Returns ``[]`` when nothing valid can be extracted. | |
| """ | |
| text = _strip_code_fences(raw_response) | |
| if not text: | |
| return [] | |
| parsed: Any = None | |
| try: | |
| parsed = json.loads(text) | |
| except (json.JSONDecodeError, TypeError): | |
| # Last resort: pull the first [...] block out of mixed prose. | |
| match = re.search(r"\[[^\[\]]*\]", text) | |
| if match: | |
| try: | |
| parsed = json.loads(match.group(0)) | |
| except json.JSONDecodeError: | |
| parsed = None | |
| if parsed is None: | |
| return [] | |
| if isinstance(parsed, list): | |
| candidates = parsed | |
| elif isinstance(parsed, dict): | |
| candidates = parsed.get("attributes") | |
| if not isinstance(candidates, list): | |
| candidates = next( | |
| (v for v in parsed.values() if isinstance(v, list)), | |
| [], | |
| ) | |
| else: | |
| candidates = [] | |
| valid = {a.lower(): a for a in CELEBA_ATTRIBUTES} | |
| attrs: list[str] = [] | |
| for candidate in candidates: | |
| if not isinstance(candidate, str): | |
| continue | |
| canonical = valid.get(candidate.strip().lower()) | |
| if canonical and canonical not in attrs: | |
| attrs.append(canonical) | |
| return attrs | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # cascadeflow agent: security alert generator | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async def generate_alert( | |
| attributes: list[str], | |
| similarity: float, | |
| ) -> dict[str, Any]: | |
| """Draft a security alert for a confirmed identity match.""" | |
| pct = round(similarity * 100, 1) | |
| attr_text = ", ".join(attributes) if attributes else "no distinguishing attributes identified" | |
| system_prompt = "You are a professional security monitoring AI assistant. Do not speculate beyond the provided attributes." | |
| user_prompt = ( | |
| f"A face recognition system has found a match with {pct}% confidence. " | |
| f"The matched subject has the following attributes: {attr_text}. " | |
| f"Draft a concise, professional security alert in 2-3 sentences. " | |
| f"Include the confidence level, key attributes, and a recommended action." | |
| ) | |
| t0 = time.monotonic() | |
| deadline = t0 + _LLM_HARD_TIMEOUT_MS / 1000.0 | |
| response_text = "" | |
| provider_used = "Gemini" | |
| cost_usd = 0.0 | |
| try: | |
| # π’ ATTEMPT 1: GEMINI | |
| async with _cf_run(budget=_ALERT_BUDGET_USD, max_latency_ms=_LATENCY_LIMIT_MS): | |
| response = await _await_with_deadline( | |
| _get_client().aio.models.generate_content( | |
| model=GEMINI_MODEL, | |
| contents=f"{system_prompt}\n\n{user_prompt}", | |
| # Gemini 2.5 counts hidden "thinking" tokens against this | |
| # budget. 256 produced alerts truncated mid-sentence; 1024 | |
| # leaves room for both reasoning and the 2-3 sentence answer. | |
| config=genai_types.GenerateContentConfig(temperature=0.3, max_output_tokens=1024), | |
| ), | |
| deadline, | |
| ) | |
| response_text = _safe_text(response.text) | |
| if not response_text: | |
| raise RuntimeError("Gemini returned an empty response.") | |
| cost_usd = round( | |
| getattr(getattr(response, "usage_metadata", None), "total_token_count", 0) | |
| * 0.075 / 1_000_000, | |
| 8, | |
| ) | |
| except Exception as gemini_err: | |
| logger.warning("Gemini API failed: %s. Falling back to Groq...", gemini_err) | |
| try: | |
| # π‘ ATTEMPT 2: GROQ FALLBACK | |
| provider_used = "Groq" | |
| chat_completion = await _await_with_deadline( | |
| _get_groq_client().chat.completions.create( | |
| messages=[ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt}, | |
| ], | |
| model=GROQ_MODEL, | |
| temperature=0.3, | |
| # Reasoning-capable Groq models (gpt-oss) spend part of this | |
| # budget on hidden reasoning; 256 yielded empty content. | |
| max_tokens=1024, | |
| ), | |
| deadline, | |
| ) | |
| response_text = _safe_text(chat_completion.choices[0].message.content) | |
| if not response_text: | |
| raise RuntimeError("Groq returned an empty response.") | |
| except Exception as groq_err: | |
| logger.warning("Groq API failed: %s. Falling back to Cohere...", groq_err) | |
| try: | |
| # π ATTEMPT 3: COHERE FALLBACK (async client, awaited) | |
| provider_used = "Cohere" | |
| co_response = await _await_with_deadline( | |
| _get_cohere_client().chat( | |
| model=COHERE_MODEL, | |
| messages=[ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": user_prompt}, | |
| ], | |
| temperature=0.3, | |
| ), | |
| deadline, | |
| ) | |
| response_text = _cohere_text(co_response) | |
| if not response_text: | |
| raise RuntimeError("Cohere returned an empty response.") | |
| except Exception as cohere_err: | |
| # π΄ ATTEMPT 4: HARD FALLBACK (zero crashes) | |
| logger.error("Cohere API failed: %s. Using offline fallback.", cohere_err) | |
| provider_used = "Offline System" | |
| response_text = ( | |
| f"[SYSTEM FALLBACK ALERT] Match identified with {pct}% confidence. " | |
| f"Subject tags identified: {attr_text}. " | |
| f"Action Recommended: Verify subject credentials at control point immediately." | |
| ) | |
| latency_ms = (time.monotonic() - t0) * 1000 | |
| return { | |
| "agent_report": f"[{provider_used} Engine] {response_text}", | |
| "cascadeflow_metrics": { | |
| "latency_ms": round(latency_ms, 1), | |
| "cost_usd": cost_usd, | |
| }, | |
| } | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # cascadeflow agent: natural-language β attribute extractor | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async def extract_attributes_from_query(query: str) -> dict[str, Any]: | |
| """ | |
| Parse an operator's natural-language query into CelebA attribute names. | |
| Uses multi-provider fallback: Gemini β Groq β Cohere β keyword matcher. | |
| Returns | |
| ------- | |
| dict with keys: | |
| attributes : list[str] β matched CelebA attribute names | |
| cascadeflow_metrics: dict β latency_ms, cost_usd | |
| """ | |
| prompt = ( | |
| f"You are a database query assistant. Your only job is to map a natural " | |
| f"language description of a person to a subset of these exact attribute names:\n" | |
| f"{_ATTR_LIST_STR}\n\n" | |
| f'User description: "{query}"\n\n' | |
| f"Return ONLY a JSON array of matching attribute name strings from the list above. " | |
| f'Example: ["Male", "Eyeglasses", "Black_Hair"]. ' | |
| f"Do not include attributes that are not in the list. Return an empty array if none match." | |
| ) | |
| t0 = time.monotonic() | |
| deadline = t0 + _LLM_HARD_TIMEOUT_MS / 1000.0 | |
| raw_response = "" | |
| provider_used = "Gemini" | |
| cost_usd = 0.0 | |
| try: | |
| # π’ ATTEMPT 1: GEMINI | |
| async with _cf_run( | |
| budget=_SEARCH_BUDGET_USD, | |
| max_latency_ms=_LATENCY_LIMIT_MS, | |
| ): | |
| response = await _await_with_deadline( | |
| _get_client().aio.models.generate_content( | |
| model=GEMINI_MODEL, | |
| contents=prompt, | |
| config=genai_types.GenerateContentConfig( | |
| temperature=0.0, | |
| # JSON array plus hidden thinking tokens. | |
| max_output_tokens=512, | |
| response_mime_type="application/json", | |
| ), | |
| ), | |
| deadline, | |
| ) | |
| raw_response = _safe_text(response.text) | |
| cost_usd = round( | |
| getattr(getattr(response, "usage_metadata", None), "total_token_count", 0) | |
| * 0.075 / 1_000_000, | |
| 8, | |
| ) | |
| if not raw_response: | |
| raise RuntimeError("Gemini returned an empty response.") | |
| except Exception as gemini_err: | |
| logger.warning( | |
| "Gemini API failed for attribute extraction: %s. Falling back to Groq...", | |
| gemini_err, | |
| ) | |
| try: | |
| # π‘ ATTEMPT 2: GROQ FALLBACK | |
| provider_used = "Groq" | |
| chat_completion = await _await_with_deadline( | |
| _get_groq_client().chat.completions.create( | |
| messages=[ | |
| {"role": "system", "content": "You are a database query assistant. Return only valid JSON."}, | |
| {"role": "user", "content": prompt}, | |
| ], | |
| model=GROQ_MODEL, | |
| temperature=0.0, | |
| # JSON answer plus hidden reasoning tokens. | |
| max_tokens=512, | |
| ), | |
| deadline, | |
| ) | |
| raw_response = _safe_text(chat_completion.choices[0].message.content) | |
| if not raw_response: | |
| raise RuntimeError("Groq returned an empty response.") | |
| except Exception as groq_err: | |
| logger.warning( | |
| "Groq API failed for attribute extraction: %s. Falling back to Cohere...", | |
| groq_err, | |
| ) | |
| try: | |
| # π ATTEMPT 3: COHERE FALLBACK (async client, awaited) | |
| provider_used = "Cohere" | |
| co_response = await _await_with_deadline( | |
| _get_cohere_client().chat( | |
| model=COHERE_MODEL, | |
| messages=[ | |
| {"role": "system", "content": "You are a database query assistant. Return only valid JSON arrays."}, | |
| {"role": "user", "content": prompt}, | |
| ], | |
| temperature=0.0, | |
| ), | |
| deadline, | |
| ) | |
| raw_response = _cohere_text(co_response) | |
| if not raw_response: | |
| raise RuntimeError("Cohere returned an empty response.") | |
| except Exception as cohere_err: | |
| # π΄ ATTEMPT 4: MANUAL KEYWORD MATCHING FALLBACK | |
| logger.error( | |
| "Cohere API failed for attribute extraction: %s. Using keyword matching.", | |
| cohere_err, | |
| ) | |
| provider_used = "Keyword Matcher" | |
| raw_response = "" | |
| # ββ Parse the response; fall back to keywords if it was empty/invalid βββββ | |
| attrs = _parse_attributes(raw_response) | |
| keyword_attrs, excluded_attrs = _keyword_filter(query) | |
| if not attrs: | |
| if raw_response: | |
| logger.warning( | |
| "%s returned no usable attributes; using keyword matcher. Response: %s", | |
| provider_used, | |
| raw_response[:200], | |
| ) | |
| else: | |
| logger.info("Using keyword matcher for attribute extraction.") | |
| provider_used = "Keyword Matcher" | |
| attrs = keyword_attrs | |
| else: | |
| # Even a good provider response can omit a negative constraint | |
| # ("woman" β provider returns Smiling but no way to exclude Male) or | |
| # contradict the query ("woman" β provider returns Male). Apply the | |
| # deterministic exclusions before returning. | |
| excluded_lower = {a.lower() for a in excluded_attrs} | |
| attrs = [a for a in attrs if a.lower() not in excluded_lower] | |
| if not attrs: | |
| provider_used = "Keyword Matcher" | |
| attrs = keyword_attrs | |
| latency_ms = (time.monotonic() - t0) * 1000 | |
| logger.info( | |
| "[%s] Extracted attributes: %s (excluding %s)", | |
| provider_used, | |
| attrs, | |
| excluded_attrs, | |
| ) | |
| return { | |
| "attributes": attrs, | |
| "excluded_attributes": excluded_attrs, | |
| "cascadeflow_metrics": { | |
| "latency_ms": round(latency_ms, 1), | |
| "cost_usd": cost_usd, | |
| }, | |
| } | |