""" core/gemini_agent.py — Sentinel-AI ====================================== Wraps the Google GenAI SDK with cascadeflow guardrails to provide two async functions: • generate_alert(attributes, similarity) Drafts a professional security alert from matched face attributes. • extract_attributes_from_query(query) Parses a natural-language operator query into a list of CelebA attribute names that can be used to filter the FAISS metadata. Both functions are decorated with @cascadeflow.agent (policy metadata) and execute inside a cascadeflow.run() scope to record latency and cost budget. Every cloud provider is optional at runtime: Gemini → Groq → Cohere → deterministic local fallback. Importing this module never requires an API key; clients are created lazily on first use. """ from __future__ import annotations import asyncio import json import logging import os import re import time from typing import Any import cascadeflow from core.paths import env_float # cascadeflow.agent at the top level resolves to the agent.py module (not a # callable) due to lazy-import aliasing. The correct decorator is # cascadeflow.harness_agent, which is the harness.api.agent function. _cf_agent = cascadeflow.harness_agent _cf_run = cascadeflow.run import cohere from google import genai from google.genai import types as genai_types from groq import AsyncGroq logger = logging.getLogger(__name__) # Load fallback models from .env GROQ_MODEL = os.getenv("GROQ_MODEL", "openai/gpt-oss-120b") COHERE_MODEL = os.getenv("COHERE_MODEL", "command-r-plus-08-2024") # Instantiate fallback clients lazily _groq_client: AsyncGroq | None = None _cohere_client: cohere.AsyncClientV2 | None = None def _get_groq_client() -> AsyncGroq: global _groq_client if _groq_client is None: api_key = os.getenv("GROQ_API_KEY") if not api_key: raise RuntimeError("GROQ_API_KEY is not configured.") _groq_client = AsyncGroq(api_key=api_key) return _groq_client def _get_cohere_client() -> cohere.AsyncClientV2: global _cohere_client if _cohere_client is None: api_key = os.getenv("COHERE_API_KEY") if not api_key: raise RuntimeError("COHERE_API_KEY is not configured.") # AsyncClientV2 is required: the sync client would block the event loop. _cohere_client = cohere.AsyncClientV2(api_key=api_key) return _cohere_client def warm_providers() -> list[str]: """ Initialise every configured provider client at startup. Client construction is synchronous (auth discovery, HTTP pool setup) and otherwise lands on the first live request, where it can add ~1 s and push an otherwise in-budget call over the deadline. Missing keys are skipped. """ warmed: list[str] = [] for name, factory in ( ("Gemini", _get_client), ("Groq", _get_groq_client), ("Cohere", _get_cohere_client), ): try: factory() warmed.append(name) except Exception as exc: logger.info("%s client not warmed: %s", name, exc) return warmed # ── Configuration ───────────────────────────────────────────────────────────── # Accept both GEMINI_API_KEY (per spec) and GOOGLE_API_KEY (legacy) so either # variable in .env works without breaking existing setups. GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.5-flash") # Per-call guardrail budgets (can be overridden via env) _ALERT_BUDGET_USD = float(os.getenv("CASCADEFLOW_ALERT_BUDGET", "0.01")) _SEARCH_BUDGET_USD = float(os.getenv("CASCADEFLOW_SEARCH_BUDGET", "0.005")) _LATENCY_LIMIT_MS = float(os.getenv("CASCADEFLOW_LATENCY_LIMIT_MS", "2000")) # cascadeflow records ``max_latency_ms`` but does not cancel a slow provider. # These two values are enforced with ``asyncio.wait_for``: # • LLM_HARD_TIMEOUT_MS — ceiling for the whole Gemini→Groq→Cohere chain # • LLM_PROVIDER_TIMEOUT_MS — ceiling for a single provider attempt # A slow Gemini call is cancelled at the provider deadline, leaving the rest of # the budget for a fallback. Raise LLM_HARD_TIMEOUT_MS if you prefer answer # quality over the 2 s target. _LLM_HARD_TIMEOUT_MS = max( 100.0, env_float("LLM_HARD_TIMEOUT_MS", _LATENCY_LIMIT_MS), ) _LLM_PROVIDER_TIMEOUT_MS = max( 50.0, env_float("LLM_PROVIDER_TIMEOUT_MS", max(250.0, _LATENCY_LIMIT_MS * 0.6)), ) # ── Lazy Gemini client singleton ────────────────────────────────────────────── _client: genai.Client | None = None def _get_client() -> genai.Client: """Return the Gemini client, initialising it on first call.""" global _client if _client is None: api_key = os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") or "" if not api_key: raise RuntimeError( "No Gemini API key found. " "Set GEMINI_API_KEY in your .env file " "(copy .env.example → .env and fill in the value)." ) _client = genai.Client(api_key=api_key) return _client def _safe_text(value: Any) -> str: """Return ``value.strip()`` for strings, otherwise an empty string.""" return value.strip() if isinstance(value, str) else "" def _cohere_text(response: Any) -> str: """Extract the first text block from a Cohere V2 chat response.""" message = getattr(response, "message", None) content = getattr(message, "content", None) if not content: return "" return _safe_text(getattr(content[0], "text", None)) async def _await_with_deadline(coro, deadline: float): """ Await ``coro`` with the per-provider and whole-chain deadlines applied. ``deadline`` is a ``time.monotonic()`` value. ``asyncio.wait_for`` waits for the cancelled provider to finish cleaning up, which can itself take hundreds of milliseconds (the google-genai client closes an HTTP stream). That overran the wall-clock budget in testing, so this helper races the task against a timer instead: on expiry it cancels the task, attaches a callback to consume the eventual result, and raises immediately. """ remaining = deadline - time.monotonic() if remaining <= 0: coro.close() raise asyncio.TimeoutError("LLM hard deadline exceeded.") provider_timeout = min(remaining, _LLM_PROVIDER_TIMEOUT_MS / 1000.0) task = asyncio.ensure_future(coro) done, _pending = await asyncio.wait({task}, timeout=provider_timeout) if task in done: return task.result() task.cancel() def _consume_result(finished: asyncio.Future) -> None: # Retrieve the exception so asyncio does not log "exception was never # retrieved" when the provider finishes cancelling in the background. if not finished.cancelled(): try: finished.exception() except BaseException: pass task.add_done_callback(_consume_result) raise asyncio.TimeoutError("LLM hard deadline exceeded.") # ── Valid CelebA attribute names (used in the extraction prompt) ────────────── CELEBA_ATTRIBUTES = [ "5_o_Clock_Shadow", "Arched_Eyebrows", "Attractive", "Bags_Under_Eyes", "Bald", "Bangs", "Big_Lips", "Big_Nose", "Black_Hair", "Blond_Hair", "Blurry", "Brown_Hair", "Bushy_Eyebrows", "Chubby", "Double_Chin", "Eyeglasses", "Goatee", "Gray_Hair", "Heavy_Makeup", "High_Cheekbones", "Male", "Mouth_Slightly_Open", "Mustache", "Narrow_Eyes", "No_Beard", "Oval_Face", "Pale_Skin", "Pointy_Nose", "Receding_Hairline", "Rosy_Cheeks", "Sideburns", "Smiling", "Straight_Hair", "Wavy_Hair", "Wearing_Earrings", "Wearing_Hat", "Wearing_Lipstick", "Wearing_Necklace", "Wearing_Necktie", "Young", ] _ATTR_LIST_STR = ", ".join(CELEBA_ATTRIBUTES) # ───────────────────────────────────────────────────────────────────────────── # Local keyword fallback # ───────────────────────────────────────────────────────────────────────────── # Ordered longest-phrase-first so "black hair" wins over a bare "hair" rule. # Generic "beard" is intentionally absent: CelebA has no positive "Beard" # attribute and mapping it to the inverse "No_Beard" returned beardless # subjects. "No beard" is handled explicitly below. _KEYWORD_MAP: tuple[tuple[str, str], ...] = ( ("5 o'clock shadow", "5_o_Clock_Shadow"), ("five o'clock shadow", "5_o_Clock_Shadow"), ("black hair", "Black_Hair"), ("blond hair", "Blond_Hair"), ("blonde hair", "Blond_Hair"), ("brown hair", "Brown_Hair"), ("gray hair", "Gray_Hair"), ("grey hair", "Gray_Hair"), ("straight hair", "Straight_Hair"), ("rosy cheeks", "Rosy_Cheeks"), ("receding hairline", "Receding_Hairline"), ("bushy eyebrows", "Bushy_Eyebrows"), ("arched eyebrows", "Arched_Eyebrows"), ("bags under eyes", "Bags_Under_Eyes"), ("bags under the eyes", "Bags_Under_Eyes"), ("double chin", "Double_Chin"), ("big lips", "Big_Lips"), ("big nose", "Big_Nose"), ("pointy nose", "Pointy_Nose"), ("high cheekbones", "High_Cheekbones"), ("heavy makeup", "Heavy_Makeup"), ("mouth slightly open", "Mouth_Slightly_Open"), ("pale skin", "Pale_Skin"), ("wearing a hat", "Wearing_Hat"), ("hat", "Wearing_Hat"), ("wearing a necklace", "Wearing_Necklace"), ("wearing a necktie", "Wearing_Necktie"), ("wearing a tie", "Wearing_Necktie"), ("earrings", "Wearing_Earrings"), ("necklace", "Wearing_Necklace"), ("necktie", "Wearing_Necktie"), ("eyeglasses", "Eyeglasses"), ("glasses", "Eyeglasses"), ("spectacles", "Eyeglasses"), ("blurry", "Blurry"), ("blurred", "Blurry"), ("mustache", "Mustache"), ("moustache", "Mustache"), ("goatee", "Goatee"), ("sideburns", "Sideburns"), ("stubble", "5_o_Clock_Shadow"), ("bald", "Bald"), ("bangs", "Bangs"), ("fringe", "Bangs"), ("wavy", "Wavy_Hair"), ("curly", "Wavy_Hair"), ("straight hair", "Straight_Hair"), ("lipstick", "Wearing_Lipstick"), ("makeup", "Heavy_Makeup"), ("smiling", "Smiling"), ("smile", "Smiling"), ("chubby", "Chubby"), ("attractive", "Attractive"), ("pale", "Pale_Skin"), ) _FEMALE_WORDS = ("woman", "women", "female", "girl", "lady", "she", "her", "hers") _MALE_WORDS = ("man", "men", "male", "boy", "guy", "gentleman", "he", "him", "his") _YOUNG_WORDS = ("young", "youth", "teen", "teenager", "teenage", "child", "kid") _OLD_WORDS = ("old", "older", "elderly", "senior", "aged", "middle-aged") def _has_words(text: str, words: tuple[str, ...]) -> bool: """Word-boundary match to avoid 'he' inside 'there' / 'her' inside 'other'.""" pattern = r"\b(?:" + "|".join(re.escape(w) for w in words) + r")\b" return re.search(pattern, text, flags=re.IGNORECASE) is not None # Looks backwards from a keyword for a nearby negation: # "no glasses", "without a hat", "not wearing makeup", # "doesn't have a beard", "free of earrings". _NEGATION_RE = re.compile( r"\b(?:no|without|not(?:\s+(?:wearing|have|having))?|" r"isn't|aren't|doesn't(?:\s+have)?|don't|dont|zero|free\s+of)" r"\s+(?:\w+\s+){0,3}$", re.IGNORECASE, ) def _words_with_negation(text: str, words: tuple[str, ...]) -> tuple[bool, bool]: """ Return ``(positive, negated)`` for a set of words. Each occurrence is classified independently, so "a woman, not a man" counts as positive-female and negated-male. """ pattern = r"\b(?:" + "|".join(re.escape(w) for w in words) + r")\b" positive = negated = False for match in re.finditer(pattern, text, flags=re.IGNORECASE): if _NEGATION_RE.search(text[: match.start()]): negated = True else: positive = True return positive, negated def _keyword_filter(query: str) -> tuple[list[str], list[str]]: """ Deterministic offline extraction of required *and excluded* attributes. CelebA encodes gender and age as single bits (``Male``, ``Young``); there is no ``Female`` or ``Old`` column. A query for "woman" therefore cannot be expressed as a positive filter alone — returning every face that is smiling and blond includes men. Negative constraints close that gap. Negated keywords ("no glasses", "without a hat") are likewise moved to the exclusion list instead of being added as positive requirements, which is the same class of bug as mapping "beard" to ``No_Beard``. """ query_lower = query.lower() attrs: list[str] = [] excluded: list[str] = [] def add(bucket: list[str], attr: str) -> None: if attr not in bucket: bucket.append(attr) for keyword, attr in _KEYWORD_MAP: for match in re.finditer(rf"\b{re.escape(keyword)}\b", query_lower): if _NEGATION_RE.search(query_lower[: match.start()]): add(excluded, attr) else: add(attrs, attr) male_pos, male_neg = _words_with_negation(query_lower, _MALE_WORDS) female_pos, female_neg = _words_with_negation(query_lower, _FEMALE_WORDS) if female_pos and not male_pos: add(excluded, "Male") elif male_pos and not female_pos: add(attrs, "Male") elif male_neg and not female_neg: # "not a man" / "no men" — non-male is the intended constraint. add(excluded, "Male") young_pos, young_neg = _words_with_negation(query_lower, _YOUNG_WORDS) old_pos, old_neg = _words_with_negation(query_lower, _OLD_WORDS) if old_pos and not young_pos: add(excluded, "Young") elif young_pos and not old_pos: add(attrs, "Young") elif young_neg and not old_neg: add(excluded, "Young") # "no beard"/"without a beard"/"beardless" is the only correct way to use # No_Beard (CelebA has no positive "Beard" label). if re.search(r"\b(?:no|without(?:\s+a)?)\s+beard\b|\bbeardless\b", query_lower): add(attrs, "No_Beard") # "clean-shaven" implies all three facial-hair labels are absent. if re.search(r"\bclean[- ]shaven\b", query_lower): add(attrs, "No_Beard") for attr in ("Mustache", "Goatee", "5_o_Clock_Shadow"): add(excluded, attr) # An attribute cannot be both required and forbidden. excluded_set = {a.lower() for a in excluded} attrs = [a for a in attrs if a.lower() not in excluded_set] return ( [a for a in attrs if a in CELEBA_ATTRIBUTES], [a for a in excluded if a in CELEBA_ATTRIBUTES], ) def _keyword_extract(query: str) -> list[str]: """Backward-compatible positive-only view of :func:`_keyword_filter`.""" return _keyword_filter(query)[0] def _strip_code_fences(text: str) -> str: """Remove ```json … ``` wrappers that models sometimes emit.""" text = text.strip() if text.startswith("```"): text = re.sub(r"^```[a-zA-Z0-9_-]*\s*", "", text) text = re.sub(r"\s*```$", "", text) return text.strip() def _parse_attributes(raw_response: str) -> list[str]: """ Parse an LLM response into validated CelebA attribute names. Accepts a bare JSON array, ``{"attributes": [...]}``, any object containing an array, or a JSON array embedded in prose/code fences. Returns ``[]`` when nothing valid can be extracted. """ text = _strip_code_fences(raw_response) if not text: return [] parsed: Any = None try: parsed = json.loads(text) except (json.JSONDecodeError, TypeError): # Last resort: pull the first [...] block out of mixed prose. match = re.search(r"\[[^\[\]]*\]", text) if match: try: parsed = json.loads(match.group(0)) except json.JSONDecodeError: parsed = None if parsed is None: return [] if isinstance(parsed, list): candidates = parsed elif isinstance(parsed, dict): candidates = parsed.get("attributes") if not isinstance(candidates, list): candidates = next( (v for v in parsed.values() if isinstance(v, list)), [], ) else: candidates = [] valid = {a.lower(): a for a in CELEBA_ATTRIBUTES} attrs: list[str] = [] for candidate in candidates: if not isinstance(candidate, str): continue canonical = valid.get(candidate.strip().lower()) if canonical and canonical not in attrs: attrs.append(canonical) return attrs # ───────────────────────────────────────────────────────────────────────────── # cascadeflow agent: security alert generator # ───────────────────────────────────────────────────────────────────────────── @_cf_agent(budget=_ALERT_BUDGET_USD) async def generate_alert( attributes: list[str], similarity: float, ) -> dict[str, Any]: """Draft a security alert for a confirmed identity match.""" pct = round(similarity * 100, 1) attr_text = ", ".join(attributes) if attributes else "no distinguishing attributes identified" system_prompt = "You are a professional security monitoring AI assistant. Do not speculate beyond the provided attributes." user_prompt = ( f"A face recognition system has found a match with {pct}% confidence. " f"The matched subject has the following attributes: {attr_text}. " f"Draft a concise, professional security alert in 2-3 sentences. " f"Include the confidence level, key attributes, and a recommended action." ) t0 = time.monotonic() deadline = t0 + _LLM_HARD_TIMEOUT_MS / 1000.0 response_text = "" provider_used = "Gemini" cost_usd = 0.0 try: # 🟢 ATTEMPT 1: GEMINI async with _cf_run(budget=_ALERT_BUDGET_USD, max_latency_ms=_LATENCY_LIMIT_MS): response = await _await_with_deadline( _get_client().aio.models.generate_content( model=GEMINI_MODEL, contents=f"{system_prompt}\n\n{user_prompt}", # Gemini 2.5 counts hidden "thinking" tokens against this # budget. 256 produced alerts truncated mid-sentence; 1024 # leaves room for both reasoning and the 2-3 sentence answer. config=genai_types.GenerateContentConfig(temperature=0.3, max_output_tokens=1024), ), deadline, ) response_text = _safe_text(response.text) if not response_text: raise RuntimeError("Gemini returned an empty response.") cost_usd = round( getattr(getattr(response, "usage_metadata", None), "total_token_count", 0) * 0.075 / 1_000_000, 8, ) except Exception as gemini_err: logger.warning("Gemini API failed: %s. Falling back to Groq...", gemini_err) try: # 🟡 ATTEMPT 2: GROQ FALLBACK provider_used = "Groq" chat_completion = await _await_with_deadline( _get_groq_client().chat.completions.create( messages=[ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], model=GROQ_MODEL, temperature=0.3, # Reasoning-capable Groq models (gpt-oss) spend part of this # budget on hidden reasoning; 256 yielded empty content. max_tokens=1024, ), deadline, ) response_text = _safe_text(chat_completion.choices[0].message.content) if not response_text: raise RuntimeError("Groq returned an empty response.") except Exception as groq_err: logger.warning("Groq API failed: %s. Falling back to Cohere...", groq_err) try: # 🟠 ATTEMPT 3: COHERE FALLBACK (async client, awaited) provider_used = "Cohere" co_response = await _await_with_deadline( _get_cohere_client().chat( model=COHERE_MODEL, messages=[ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], temperature=0.3, ), deadline, ) response_text = _cohere_text(co_response) if not response_text: raise RuntimeError("Cohere returned an empty response.") except Exception as cohere_err: # 🔴 ATTEMPT 4: HARD FALLBACK (zero crashes) logger.error("Cohere API failed: %s. Using offline fallback.", cohere_err) provider_used = "Offline System" response_text = ( f"[SYSTEM FALLBACK ALERT] Match identified with {pct}% confidence. " f"Subject tags identified: {attr_text}. " f"Action Recommended: Verify subject credentials at control point immediately." ) latency_ms = (time.monotonic() - t0) * 1000 return { "agent_report": f"[{provider_used} Engine] {response_text}", "cascadeflow_metrics": { "latency_ms": round(latency_ms, 1), "cost_usd": cost_usd, }, } # ───────────────────────────────────────────────────────────────────────────── # cascadeflow agent: natural-language → attribute extractor # ───────────────────────────────────────────────────────────────────────────── @_cf_agent(budget=_SEARCH_BUDGET_USD) async def extract_attributes_from_query(query: str) -> dict[str, Any]: """ Parse an operator's natural-language query into CelebA attribute names. Uses multi-provider fallback: Gemini → Groq → Cohere → keyword matcher. Returns ------- dict with keys: attributes : list[str] — matched CelebA attribute names cascadeflow_metrics: dict — latency_ms, cost_usd """ prompt = ( f"You are a database query assistant. Your only job is to map a natural " f"language description of a person to a subset of these exact attribute names:\n" f"{_ATTR_LIST_STR}\n\n" f'User description: "{query}"\n\n' f"Return ONLY a JSON array of matching attribute name strings from the list above. " f'Example: ["Male", "Eyeglasses", "Black_Hair"]. ' f"Do not include attributes that are not in the list. Return an empty array if none match." ) t0 = time.monotonic() deadline = t0 + _LLM_HARD_TIMEOUT_MS / 1000.0 raw_response = "" provider_used = "Gemini" cost_usd = 0.0 try: # 🟢 ATTEMPT 1: GEMINI async with _cf_run( budget=_SEARCH_BUDGET_USD, max_latency_ms=_LATENCY_LIMIT_MS, ): response = await _await_with_deadline( _get_client().aio.models.generate_content( model=GEMINI_MODEL, contents=prompt, config=genai_types.GenerateContentConfig( temperature=0.0, # JSON array plus hidden thinking tokens. max_output_tokens=512, response_mime_type="application/json", ), ), deadline, ) raw_response = _safe_text(response.text) cost_usd = round( getattr(getattr(response, "usage_metadata", None), "total_token_count", 0) * 0.075 / 1_000_000, 8, ) if not raw_response: raise RuntimeError("Gemini returned an empty response.") except Exception as gemini_err: logger.warning( "Gemini API failed for attribute extraction: %s. Falling back to Groq...", gemini_err, ) try: # 🟡 ATTEMPT 2: GROQ FALLBACK provider_used = "Groq" chat_completion = await _await_with_deadline( _get_groq_client().chat.completions.create( messages=[ {"role": "system", "content": "You are a database query assistant. Return only valid JSON."}, {"role": "user", "content": prompt}, ], model=GROQ_MODEL, temperature=0.0, # JSON answer plus hidden reasoning tokens. max_tokens=512, ), deadline, ) raw_response = _safe_text(chat_completion.choices[0].message.content) if not raw_response: raise RuntimeError("Groq returned an empty response.") except Exception as groq_err: logger.warning( "Groq API failed for attribute extraction: %s. Falling back to Cohere...", groq_err, ) try: # 🟠 ATTEMPT 3: COHERE FALLBACK (async client, awaited) provider_used = "Cohere" co_response = await _await_with_deadline( _get_cohere_client().chat( model=COHERE_MODEL, messages=[ {"role": "system", "content": "You are a database query assistant. Return only valid JSON arrays."}, {"role": "user", "content": prompt}, ], temperature=0.0, ), deadline, ) raw_response = _cohere_text(co_response) if not raw_response: raise RuntimeError("Cohere returned an empty response.") except Exception as cohere_err: # 🔴 ATTEMPT 4: MANUAL KEYWORD MATCHING FALLBACK logger.error( "Cohere API failed for attribute extraction: %s. Using keyword matching.", cohere_err, ) provider_used = "Keyword Matcher" raw_response = "" # ── Parse the response; fall back to keywords if it was empty/invalid ───── attrs = _parse_attributes(raw_response) keyword_attrs, excluded_attrs = _keyword_filter(query) if not attrs: if raw_response: logger.warning( "%s returned no usable attributes; using keyword matcher. Response: %s", provider_used, raw_response[:200], ) else: logger.info("Using keyword matcher for attribute extraction.") provider_used = "Keyword Matcher" attrs = keyword_attrs else: # Even a good provider response can omit a negative constraint # ("woman" → provider returns Smiling but no way to exclude Male) or # contradict the query ("woman" → provider returns Male). Apply the # deterministic exclusions before returning. excluded_lower = {a.lower() for a in excluded_attrs} attrs = [a for a in attrs if a.lower() not in excluded_lower] if not attrs: provider_used = "Keyword Matcher" attrs = keyword_attrs latency_ms = (time.monotonic() - t0) * 1000 logger.info( "[%s] Extracted attributes: %s (excluding %s)", provider_used, attrs, excluded_attrs, ) return { "attributes": attrs, "excluded_attributes": excluded_attrs, "cascadeflow_metrics": { "latency_ms": round(latency_ms, 1), "cost_usd": cost_usd, }, }