feat: token compression, geocoding, 18-tool layer, advisory prompt rule

Phase 1 and 2 - Python middleware + tool restructure:
- compress_tool_output(): metadata hoisting + null stripping + CSV rendering
Verified 90.4% token reduction (7,360 -> 704 tokens on failing trace)
- geocode_address_handler(): Mapbox API with timeout/HTTP/parse error handling
- SYNTHESIS_PROMPT: Rule 6 - advisory two-part format + mandatory disclaimer
- Session history cap: 20 -> 6 messages
- tools.py: 18 tools total, retired get_properties_by_filter, added 8 new tools,
rewrote all descriptions for LLM disambiguation
- supabase_service.py: clamping for all RPCs, Python-side tool guard
- config.py: added MAPBOX_ACCESS_TOKEN
- requirements.txt: added requests>=2.31.0
- .gitignore: added additional_tools.py

.gitignore

@@ -1,76 +1,77 @@
-venv312/
-venv*/
-.venv/
-tests/
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-outputs/
-onnx_model/
-reports/
-sample_data/
-# C extensions
-*.so
-inst/
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-docs_research
-tools
-Real Estate Intelligence Layer.md
-grant_privileges.sql
-agent_prompt.md
-logs/
-dynamic_tool_calling_features.txt
-Frontend_API_Updates.md
-*.npy
-section_titles.json
-test_rpcs.py
-test_api.py
-misc
-test_langfuse.py
+venv312/
+venv*/
+.venv/
+tests/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+outputs/
+onnx_model/
+reports/
+sample_data/
+# C extensions
+*.so
+inst/
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+docs_research
+tools
+Real Estate Intelligence Layer.md
+grant_privileges.sql
+agent_prompt.md
+logs/
+dynamic_tool_calling_features.txt
+Frontend_API_Updates.md
+*.npy
+section_titles.json
+test_rpcs.py
+test_api.py
+misc
+test_langfuse.py
+additional_tools.py

config.py

@@ -11,3 +11,5 @@ APP_WRITE_PROJECT_ID = os.environ.get("APP_WRITE_PROJECT_ID", "")
 APP_WRITE_API_ENDPOINT = os.environ.get("APP_WRITE_API_ENDPOINT", "")
 APP_WRITE_API_KEY = os.environ.get("APP_WRITE_API_KEY", "")
 APP_WRITE_BUCKET_ID = os.environ.get("APP_WRITE_BUCKET_ID", "")
+
+MAPBOX_ACCESS_TOKEN = os.environ.get("MAPBOX_ACCESS_TOKEN", "")

requirements.txt

@@ -1,12 +1,13 @@
-fastapi==0.115.0
-uvicorn[standard]==0.32.0
-sentence-transformers==3.3.1
-numpy==1.26.4
-groq==0.13.0
-python-dotenv==1.0.1
-supabase==2.7.4
-appwrite==22.2.0
-pydantic==2.13.4
-httpx==0.27.2
-langfuse>=2.0.0
-openinference-instrumentation-groq
+fastapi==0.115.0
+uvicorn[standard]==0.32.0
+sentence-transformers==3.3.1
+numpy==1.26.4
+groq==0.13.0
+python-dotenv==1.0.1
+supabase==2.7.4
+appwrite==22.2.0
+pydantic==2.13.4
+httpx==0.27.2
+requests>=2.31.0
+langfuse>=2.0.0
+openinference-instrumentation-groq

services/groq_service.py

@@ -1,14 +1,17 @@
+import csv
+import io
 import json
 import re
 import os
 import time
+import requests
 import numpy as np
 from groq import Groq
 from services.tools import REAL_ESTATE_TOOLS
 from services.supabase_service import execute_tool_rpc, search_methodology_rag
 from services.embedding_service import get_embedding_model
 from services.observability import setup_logger
-from config import GROQ_API_KEY
+from config import GROQ_API_KEY, MAPBOX_ACCESS_TOKEN
 from langfuse import observe, propagate_attributes, get_client
 import langfuse
 
@@ -53,7 +56,7 @@ IMMUTABLE SYSTEM BOUNDARIES & HARD FACTS:
 1. TRACKED MARKETS: You ONLY track two markets: 'NYC/NJ Metro' and 'Miami'. 
 2. TRACKED PLATFORMS: You ONLY track two platforms: 'Airbnb' (Active daily tracking) and 'Vrbo' (Historical data only).
 3. ABSOLUTE FORBIDDEN ENTITIES: You must NEVER list, suggest, or mention any other cities (e.g., Los Angeles, Chicago, Houston, Orlando) or other booking platforms (e.g., Booking.com, Expedia, Tripadvisor). If asked about them, state plainly that they are outside Joule Dynamics' current tracking scope.
-4. ZERO FABRICATION: Every single price, rate change percentage, property count, and availability status MUST come directly from a returned tool output JSON. If a tool returns no data or an error, state: "I don't have that information in the current real estate scope."
+4. ZERO FABRICATION: Every single price, rate change percentage, property count, and availability status MUST come directly from a returned tool output. If a tool returns no data or an error, state: "I don't have that information in the current real estate scope."
 5. NO RAW SQL: Never attempt to write or generate SQL queries. Rely strictly on the registered tool RPCs provided.
 
 OPERATIONAL RULES:
@@ -65,8 +68,180 @@ OPERATIONAL RULES:
 ```json
 {"clarification_options": ["Option A", "Option B"]}
 ```
+6. ADVISORY & STRATEGY RESPONSES: When a user asks for pricing recommendations, competitive strategy, or "what should I do?" guidance, you MUST use a strict two-part structure:
+
+   **What the data shows:** (grounded section)
+   Present only facts derived directly from tool outputs. Use precise numbers. Format as a table or bullet list.
+
+   **Suggested approach (data-informed):** (reasoned section)
+   Offer strategic interpretation based on the data patterns above. Be specific but clearly frame this as inference from data, not a certainty.
+
+   Every advisory response MUST close with this disclaimer on its own line:
+   > ⚠️ *This is a data-informed observation, not professional pricing or financial advice. Consult a revenue management specialist for investment decisions.*
+
+   NEVER present a strategic recommendation with the same flat, factual confidence as a queried data point. The boundary between retrieved fact and model reasoning must always be explicit and visible to the user.
 """
 
+# ─── PAYLOAD COMPRESSOR ────────────────────────────────────────────────────────
+# MAX rows the LLM receives from any single tool call. Beyond this, data is
+# sliced and the LLM is told to recommend narrowing filters.
+_MAX_ROWS = 50
+
+
+def compress_tool_output(func_name: str, db_result: dict) -> str:
+    """
+    Converts raw tool RPC responses into a token-efficient string for LLM context.
+    
+    Three-step compression pipeline:
+    1. Metadata hoisting — keys with identical values across all rows extracted
+       to a single header line, removing them from every row.
+    2. Null stripping — any key with a null/None value in a row is omitted.
+    3. CSV rendering — remaining data written as CSV (headers once, values compact).
+    
+    For non-tabular (scalar/dict) results, returns a minimal string representation.
+    Estimated reduction: 70–90% vs raw JSON for time-series data.
+    """
+    if db_result.get("status") != "success":
+        msg = db_result.get("message", "unknown error")
+        return f"Tool '{func_name}' error: {msg}"
+
+    data = db_result.get("data")
+
+    # ── Scalar / single-object results ──────────────────────────────────────
+    if data is None:
+        return f"Tool '{func_name}': no data returned."
+
+    if isinstance(data, (str, int, float, bool)):
+        return f"Tool '{func_name}' result: {data}"
+
+    if isinstance(data, dict):
+        # Already a compact object (e.g. get_rate_anomaly_report, get_market_trend)
+        # Just serialise cleanly — these are already small
+        return f"Tool '{func_name}' result:\n{json.dumps(data, default=str)}"
+
+    if not isinstance(data, list) or len(data) == 0:
+        return f"Tool '{func_name}': empty result."
+
+    # ── List of rows ─────────────────────────────────────────────────────────
+    # Filter out rows where everything is None (pure null rows add no signal)
+    data = [row for row in data if isinstance(row, dict) and any(v is not None for v in row.values())]
+
+    if not data:
+        return f"Tool '{func_name}': all returned rows were empty."
+
+    # Truncate before processing to cap token exposure
+    truncated = False
+    if len(data) > _MAX_ROWS:
+        data = data[:_MAX_ROWS]
+        truncated = True
+
+    # ── Step 1: Metadata hoisting ────────────────────────────────────────────
+    all_keys = list(data[0].keys())
+    hoisted = {}
+    row_keys = []
+    for key in all_keys:
+        unique_values = {str(row.get(key)) for row in data}
+        if len(unique_values) == 1:
+            val = data[0].get(key)
+            if val is not None:
+                hoisted[key] = val
+        else:
+            row_keys.append(key)
+
+    header_parts = [f"{k}={v}" for k, v in hoisted.items()]
+    header_str = f"[{', '.join(header_parts)}]\n" if header_parts else ""
+
+    # ── Step 2 & 3: Null-strip + CSV ─────────────────────────────────────────
+    # Determine which row-level keys have at least one non-null value across all rows
+    active_keys = [k for k in row_keys if any(row.get(k) is not None for row in data)]
+
+    buf = io.StringIO()
+    writer = csv.writer(buf)
+    writer.writerow(active_keys)
+    for row in data:
+        writer.writerow([row.get(k) for k in active_keys])
+
+    truncation_notice = (
+        f"\n[Truncated to {_MAX_ROWS} rows. Advise user to narrow date range or add filters.]"
+        if truncated else ""
+    )
+    return f"{header_str}{buf.getvalue().strip()}{truncation_notice}"
+
+
+# ─── GEOCODE HANDLER (Python-side, Mapbox API) ────────────────────────────────
+
+def geocode_address_handler(address: str) -> dict:
+    """
+    Resolves a free-text address to lat/lng via the Mapbox Geocoding API.
+    Handles network failures, API errors, and empty results gracefully.
+    US-only results, single best match returned.
+    """
+    if not MAPBOX_ACCESS_TOKEN:
+        return {
+            "status": "error",
+            "message": "We are unable to geocode addresses at this time — the mapping service is not configured."
+        }
+    if not address or not address.strip():
+        return {"status": "error", "message": "No address was provided to geocode."}
+
+    url = "https://api.mapbox.com/search/geocode/v6/forward"
+    params = {
+        "q": address.strip(),
+        "access_token": MAPBOX_ACCESS_TOKEN,
+        "country": "US",
+        "limit": 1
+    }
+
+    try:
+        response = requests.get(url, params=params, timeout=6)
+        response.raise_for_status()
+        data = response.json()
+
+        features = data.get("features", [])
+        if not features:
+            return {
+                "status": "error",
+                "message": f"We are unable to locate coordinates for '{address}' at this time. Please try a more specific address or city name."
+            }
+
+        feature = features[0]
+        lon, lat = feature["geometry"]["coordinates"]
+        resolved = feature.get("properties", {}).get("full_address", address)
+        return {
+            "status": "success",
+            "latitude": round(lat, 6),
+            "longitude": round(lon, 6),
+            "resolved_address": resolved
+        }
+
+    except requests.exceptions.Timeout:
+        logger.error(f"Mapbox geocode timeout for address: {address}")
+        return {
+            "status": "error",
+            "message": "We are unable to geocode this address at this time — the mapping service timed out. Please try again shortly."
+        }
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"Mapbox geocode HTTP error {e.response.status_code} for: {address}")
+        return {
+            "status": "error",
+            "message": "We are unable to geocode this address at this time due to a service error. Please try again later."
+        }
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Mapbox geocode request failed for '{address}': {e}")
+        return {
+            "status": "error",
+            "message": "We are unable to reach the mapping service at this time. Please try again later."
+        }
+    except (KeyError, IndexError, ValueError) as e:
+        logger.error(f"Mapbox geocode parsing error for '{address}': {e}")
+        return {
+            "status": "error",
+            "message": f"We are unable to parse the location for '{address}'. Please try a more specific address."
+        }
+
+
+# ─── MAIN CHAT HANDLER ────────────────────────────────────────────────────────
+
 @observe(name="process-chat")
 async def process_chat_message(user_query: str, session_id: str, session_context: dict) -> dict:
     global section_title_embeddings, section_titles
@@ -101,7 +276,7 @@ async def process_chat_message(user_query: str, session_id: str, session_context
     router_sys_prompt = ROUTER_PROMPT + pre_check_hint
     router_messages = [{"role": "system", "content": router_sys_prompt}]
     
-    # To save tokens on routing, only include the last 4 messages of history
+    # Router only needs last 4 history messages — cheap model, keep it lean
     router_messages.extend(session_history[session_id][-4:])
     router_messages.append({"role": "user", "content": user_query})
 
@@ -114,7 +289,6 @@ async def process_chat_message(user_query: str, session_id: str, session_context
     
     routing = json.loads(router_res.choices[0].message.content)
     
-    # Handle cases where the LLM might unexpectedly return a JSON array instead of an object
     if isinstance(routing, list) and len(routing) > 0:
         routing = routing[0]
     elif not isinstance(routing, dict):
@@ -126,7 +300,6 @@ async def process_chat_message(user_query: str, session_id: str, session_context
         classification = "PATH_A"
 
     get_client().update_current_span(metadata={"classification": classification, "reason": reason})
-    snapshot_history = session_history[session_id].copy() if session_id in session_history else []
 
     # Guardrail: Immediate short-circuit if Out of Scope
     if classification == "OUT_OF_SCOPE":
@@ -140,7 +313,7 @@ async def process_chat_message(user_query: str, session_id: str, session_context
         }
 
     if classification == "GREETING":
-        reply_greeting = "Hello! I'm the Joule Dynamics Real Estate Intelligence Assistant. I can help you with rate spikes, market trends, and availability data. How can I assist you today?"
+        reply_greeting = "Hello! I'm the Joule Dynamics Real Estate Intelligence Assistant. I can help you with rate spikes, market trends, property investigations, and availability data. How can I assist you today?"
         get_client().update_current_span(output=reply_greeting)
         return {
             "reply": reply_greeting,
@@ -160,7 +333,8 @@ async def process_chat_message(user_query: str, session_id: str, session_context
         {"role": "system", "content": SYNTHESIS_PROMPT}
     ]
 
-    messages.extend(session_history[session_id])
+    # Cap history at 6 messages to avoid token inflation in multi-turn sessions
+    messages.extend(session_history[session_id][-6:])
 
     user_msg_content = f"User Context Filters: {json.dumps(session_context)}\nUser Query: {user_query}"
     messages.append({"role": "user", "content": user_msg_content})
@@ -191,20 +365,29 @@ async def process_chat_message(user_query: str, session_id: str, session_context
             func_name = tool_call.function.name
             func_args = json.loads(tool_call.function.arguments)
             
+            # ── Python-side tool handlers (not routed to Supabase) ──────────
             if func_name == "generate_data_export":
                 from services.appwrite_service import upload_document_to_appwrite
                 url = await upload_document_to_appwrite(func_args.get("content", ""), func_args.get("format", "md"))
                 db_result = {"status": "success", "url": url}
+
+            elif func_name == "geocode_address":
+                # Mapbox API call handled in Python — never hits Supabase
+                db_result = geocode_address_handler(func_args.get("address", ""))
+
             else:
                 db_result = await execute_tool_rpc(func_name, func_args)
                 
             tool_results.append({"tool": func_name, "args": func_args})
 
+            # ── Payload compression before entering LLM context ─────────────
+            compressed_content = compress_tool_output(func_name, db_result)
+
             messages.append({
                 "tool_call_id": tool_call.id,
                 "role": "tool",
                 "name": func_name,
-                "content": json.dumps(db_result)
+                "content": compressed_content
             })
 
         # Second Brain Call to Synthesize Final Output
@@ -218,10 +401,10 @@ async def process_chat_message(user_query: str, session_id: str, session_context
     else:
         final_reply = response_message.content
 
-    # Track assistant reply in history
+    # Track assistant reply in history — cap at 6 to prevent token creep
     session_history[session_id].append({"role": "assistant", "content": final_reply})
-    if len(session_history[session_id]) > 20:
-        session_history[session_id] = session_history[session_id][-20:]
+    if len(session_history[session_id]) > 6:
+        session_history[session_id] = session_history[session_id][-6:]
 
     # Parse clarification options
     suggested_actions = []

services/supabase_service.py

@@ -7,32 +7,114 @@ from config import SUPABASE_URL, SUPABASE_KEY
 supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
 embedder = get_embedding_model()
 
+# Tools handled entirely in Python — never routed to Supabase
+_PYTHON_SIDE_TOOLS = {"generate_data_export", "geocode_address"}
+
+
 def is_valid_uuid(val):
     if not isinstance(val, str):
         return False
     return bool(re.match(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', val.lower()))
 
+
+def _clamp(value, lo, hi, default=None):
+    """Clamps an integer param. Returns default if value is None."""
+    if value is None:
+        return default
+    return max(lo, min(int(value), hi))
+
+
 async def execute_tool_rpc(func_name: str, args: dict) -> dict:
-    """Executes a read-only Supabase RPC matching the tool schema."""
-    # Application layer clamping and validation
+    """Executes a read-only Supabase RPC matching the tool schema.
+    
+    All parameter clamping and validation is applied here so the LLM
+    cannot craft out-of-bound inputs that would return huge datasets.
+    """
+    if func_name in _PYTHON_SIDE_TOOLS:
+        # Safety guard — these should be intercepted before reaching here
+        return {"status": "error", "message": f"Tool '{func_name}' is handled client-side and should not route to Supabase."}
+
+    # ── Per-function validation & clamping ────────────────────────────────────
+
     if func_name == "search_properties":
-        if "p_limit" in args and args["p_limit"] is not None:
-            args["p_limit"] = max(1, min(int(args["p_limit"]), 200))
+        args["p_limit"] = _clamp(args.get("p_limit"), 1, 50, default=20)
+
     elif func_name == "get_property_rate_changes":
-        if "days_param" in args and args["days_param"] is not None:
-            args["days_param"] = max(1, min(int(args["days_param"]), 90))
-        if "compare_window_days" in args and args["compare_window_days"] is not None:
-            args["compare_window_days"] = max(1, min(int(args["compare_window_days"]), 14))
+        args["days_param"] = _clamp(args.get("days_param"), 1, 30, default=14)
+        args["compare_window_days"] = _clamp(args.get("compare_window_days"), 1, 14, default=1)
+        # Validate optional date range params
+        start = args.get("start_date")
+        end = args.get("end_date")
+        if (start and not end) or (end and not start):
+            return {"status": "error", "message": "Both start_date and end_date must be provided together."}
+        # If dates provided, remove days_param to let SQL use date range mode
+        if start and end:
+            args.pop("days_param", None)
+
+    elif func_name == "get_property_rate_history":
+        args["days_param"] = _clamp(args.get("days_param"), 1, 90, default=14)
+
+    elif func_name == "get_spike_alerts":
+        args["days"] = _clamp(args.get("days"), 1, 30, default=7)
+        if "threshold" in args and args["threshold"] is not None:
+            args["threshold"] = max(5.0, min(float(args["threshold"]), 100.0))
+
     elif func_name == "get_distance_km":
         if not is_valid_uuid(args.get("property_a_id")) or not is_valid_uuid(args.get("property_b_id")):
             return {"status": "error", "message": "Invalid UUID format. Both property IDs must be valid UUIDs."}
 
+    elif func_name == "get_rate_anomaly_report":
+        args["days_param"] = _clamp(args.get("days_param"), 1, 90, default=30)
+        if "deviation_threshold" in args and args["deviation_threshold"] is not None:
+            args["deviation_threshold"] = max(5.0, min(float(args["deviation_threshold"]), 100.0))
+
+    elif func_name == "get_market_snapshot":
+        start = args.get("start_date")
+        end = args.get("end_date")
+        if not start or not end:
+            return {"status": "error", "message": "get_market_snapshot requires both start_date and end_date."}
+        # Enforce max 90-day span to prevent massive aggregations
+        try:
+            from datetime import date
+            d1 = date.fromisoformat(start)
+            d2 = date.fromisoformat(end)
+            if d2 < d1:
+                return {"status": "error", "message": "end_date must be after start_date."}
+            if (d2 - d1).days > 90:
+                return {"status": "error", "message": "Date range cannot exceed 90 days for get_market_snapshot."}
+        except ValueError:
+            return {"status": "error", "message": "Invalid date format. Use YYYY-MM-DD."}
+
+    elif func_name == "get_market_trend":
+        args["days"] = _clamp(args.get("days"), 7, 90, default=14)
+
+    elif func_name == "compare_properties":
+        ids = args.get("property_ids", [])
+        if not isinstance(ids, list) or len(ids) < 2:
+            return {"status": "error", "message": "compare_properties requires a list of 2–5 property names or UUIDs."}
+        args["property_ids"] = ids[:5]  # Hard cap at 5
+
+    elif func_name == "get_most_volatile_properties":
+        args["days"] = _clamp(args.get("days"), 7, 90, default=14)
+        args["limit"] = _clamp(args.get("limit"), 1, 10, default=5)
+
+    elif func_name == "get_recently_changed_tracking":
+        args["days"] = _clamp(args.get("days"), 1, 90, default=30)
+
+    elif func_name == "get_nearby_properties":
+        if args.get("latitude") is None or args.get("longitude") is None:
+            return {"status": "error", "message": "get_nearby_properties requires latitude and longitude coordinates. Call geocode_address first if you only have an address."}
+        args["radius_km"] = max(0.1, min(float(args.get("radius_km", 5.0)), 20.0))
+        args["limit"] = _clamp(args.get("limit"), 1, 20, default=10)
+
+    # ── Execute RPC ───────────────────────────────────────────────────────────
     try:
         res = supabase.rpc(func_name, args).execute()
         return {"status": "success", "data": res.data}
     except Exception as e:
         return {"status": "error", "message": str(e)}
 
+
 async def search_methodology_rag(query: str, top_k: int = 3) -> list:
     """Embeds query and retrieves top matching methodology documentation chunks."""
     try:

services/tools.py

@@ -1,9 +1,15 @@
+"""
+Real Estate Intelligence Layer — Tool Definitions
+Covers all registered Supabase RPCs plus Python-side handlers.
+"""
+
 REAL_ESTATE_TOOLS = [
+    # ─── DASHBOARD & MARKET OVERVIEW ─────────────────────────────────────────
     {
         "type": "function",
         "function": {
             "name": "get_dashboard_kpis",
-            "description": "Fetch overall Real Estate Rate Monitor KPIs including tracked properties count, 7D rate changes, 25%+ spikes, and scrape health.",
+            "description": "Fetch overall Real Estate Rate Monitor KPIs: tracked property count, 7-day rate changes, spike count, and scrape health status per platform. Use for general 'how is the system doing?' or market health questions.",
             "parameters": {"type": "object", "properties": {}, "required": []}
         }
     },
@@ -11,25 +17,58 @@ REAL_ESTATE_TOOLS = [
         "type": "function",
         "function": {
             "name": "get_market_averages",
-            "description": "Get average, minimum, and maximum nightly rates aggregated by regional market (e.g., 'NYC/NJ Metro' or 'Miami').",
+            "description": "Get current average, minimum, and maximum nightly rates aggregated by market (NYC/NJ Metro or Miami). Use for 'what is the average rate in X?' questions about the present moment.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "market": {"type": "string", "description": "Market region name, e.g. 'Miami' or 'NYC/NJ Metro'"}
+                    "market": {"type": "string", "description": "Market region name, e.g. 'Miami' or 'NYC/NJ Metro'. Omit to get all markets."}
                 },
                 "required": []
             }
         }
     },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_market_snapshot",
+            "description": "Get aggregate market statistics (avg/min/max rate, spike count, availability %) for a SPECIFIC HISTORICAL date range. Use when the user gives explicit start/end dates or asks about a past period (e.g. 'how was Miami during World Cup week', 'what was the market like in late July'). NOT for current/live data — use get_market_averages for that.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "market": {"type": "string", "description": "Market region name, e.g. 'Miami' or 'NYC/NJ Metro'"},
+                    "start_date": {"type": "string", "description": "Start date, YYYY-MM-DD"},
+                    "end_date": {"type": "string", "description": "End date, YYYY-MM-DD"}
+                },
+                "required": ["market", "start_date", "end_date"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_market_trend",
+            "description": "Determine whether a market's average nightly rate has been trending UP, DOWN, or STABLE over a period, and by what percentage. Use for 'is X getting more expensive?', 'is X trending up or down?' style questions. Returns a pre-computed direction — do NOT use get_market_averages for trend questions.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "market": {"type": "string", "description": "Market region name, e.g. 'Miami' or 'NYC/NJ Metro'"},
+                    "days": {"type": "integer", "default": 14, "description": "Period to analyse in days (7–90)"}
+                },
+                "required": ["market"]
+            }
+        }
+    },
+
+    # ─── SPIKE & ANOMALY DETECTION ───────────────────────────────────────────
     {
         "type": "function",
         "function": {
             "name": "get_spike_alerts",
-            "description": "Fetch properties experiencing rate volatility spikes or drops equal to or exceeding a percentage deviation threshold from their 7-day average.",
+            "description": "Fetch properties experiencing market-wide rate spikes or crashes (deviation from 7-day trailing average at or above the threshold). Use for 'which properties spiked this week?', 'any unusual drops?', 'show me volatility alerts' questions.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "threshold": {"type": "number", "default": 25.0, "description": "Minimum absolute percentage deviation from 7-day trailing average"},
+                    "threshold": {"type": "number", "default": 25.0, "description": "Minimum absolute % deviation from 7-day trailing average"},
                     "days": {"type": "integer", "default": 7, "description": "Lookback window in days"}
                 },
                 "required": []
@@ -39,13 +78,14 @@ REAL_ESTATE_TOOLS = [
     {
         "type": "function",
         "function": {
-            "name": "get_property_rate_history",
-            "description": "Retrieve historical nightly rate time series and 7-day trailing average benchmark for a specific property.",
+            "name": "get_rate_anomaly_report",
+            "description": "Investigate a SPECIFIC property's pricing anomalies. Returns pre-computed anomaly events only (significant spikes or crashes) plus a normal rate range summary. Use when the user wants to INVESTIGATE why a single property has a suspicious price, a sudden crash, or an unexplained rate. Do NOT use for general history — use get_property_rate_changes for that.",
             "parameters": {
                 "type": "object",
                 "properties": {
                     "property_search": {"type": "string", "description": "Property name or UUID"},
-                    "days": {"type": "integer", "default": 30}
+                    "days_param": {"type": "integer", "default": 30, "description": "Lookback window in days (max 90)"},
+                    "deviation_threshold": {"type": "number", "default": 25.0, "description": "Minimum absolute % deviation to qualify as anomaly"}
                 },
                 "required": ["property_search"]
             }
@@ -54,63 +94,87 @@ REAL_ESTATE_TOOLS = [
     {
         "type": "function",
         "function": {
-            "name": "get_properties_by_filter",
-            "description": "Search and filter the property rate snapshot table by market, platform, availability, or bedroom count.",
+            "name": "get_most_volatile_properties",
+            "description": "Rank tracked properties by pricing volatility (frequency and magnitude of rate swings) over a period, optionally filtered to one market. Use for 'which property is most erratic/unstable/unpredictable?' questions.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "market": {"type": "string", "description": "e.g., 'Miami' or 'NYC/NJ Metro'"},
-                    "platform": {"type": "string", "description": "e.g., 'airbnb'"},
-                    "available": {"type": "boolean", "description": "true for available now, false for unavailable"},
-                    "bedrooms": {"type": "integer", "description": "Number of bedrooms"}
+                    "market": {"type": "string", "description": "Optional market filter ('Miami' or 'NYC/NJ Metro')"},
+                    "days": {"type": "integer", "default": 14, "description": "Period to analyse in days"},
+                    "limit": {"type": "integer", "default": 5, "description": "Number of top results to return (max 10)"}
                 },
                 "required": []
             }
         }
     },
+
+    # ─── PROPERTY-SPECIFIC QUERIES ───────────────────────────────────────────
     {
         "type": "function",
         "function": {
-            "name": "generate_data_export",
-            "description": "Generate a downloadable report or data export (CSV or Markdown) based on data and metrics the user wants to keep.",
+            "name": "get_property_snapshot",
+            "description": "Get a single property's CURRENT rate, 7-day trailing average, availability status, and last scrape time in one compact response. Use for 'what is X charging right now?', 'is X available?', 'what is X's current trailing average?' questions. Much lighter than get_property_rate_changes — prefer this for current-state questions.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "format": {"type": "string", "description": "Format of the export: 'csv' or 'md'"},
-                    "content": {"type": "string", "description": "The full text content or CSV data to be exported into the file"}
+                    "property_search": {"type": "string", "description": "Property name or UUID"}
                 },
-                "required": ["format", "content"]
+                "required": ["property_search"]
             }
         }
     },
     {
         "type": "function",
         "function": {
-            "name": "get_tracked_markets",
-            "description": "Fetch active markets being tracked, optionally filtered by platform.",
+            "name": "get_property_rate_changes",
+            "description": "Show a specific property's historical rate time-series with trailing averages. Use when the user wants to SEE how rates changed over time (exploration/history) — NOT for investigating a suspicious price (use get_rate_anomaly_report) and NOT for current rate (use get_property_snapshot). Supports either a rolling lookback window OR explicit start/end dates.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "p_platform": {"type": "string", "description": "Optional platform filter (e.g. 'Airbnb' or 'Vrbo')"}
+                    "property_search": {"type": "string", "description": "Property name or UUID (required)"},
+                    "days_param": {"type": "integer", "default": 14, "description": "Rolling lookback window in days (max 30). Ignored if start_date/end_date are provided."},
+                    "compare_window_days": {"type": "integer", "default": 1, "description": "Days to compare against for pct_change_vs_prev (max 14)"},
+                    "start_date": {"type": "string", "description": "Optional explicit start date YYYY-MM-DD. Use instead of days_param when user gives a specific date range."},
+                    "end_date": {"type": "string", "description": "Optional explicit end date YYYY-MM-DD. Must be provided together with start_date."}
                 },
-                "required": []
+                "required": ["property_search"]
             }
         }
     },
+    {
+        "type": "function",
+        "function": {
+            "name": "compare_properties",
+            "description": "Compare current rate, trailing average, availability, and volatility side-by-side for 2–5 specific properties. Use for 'compare X and Y', 'how does X stack up against Y and Z?', 'which of these properties is cheapest?' questions.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "property_ids": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "2–5 property names or UUIDs to compare side-by-side"
+                    }
+                },
+                "required": ["property_ids"]
+            }
+        }
+    },
+
+    # ─── SEARCH & FILTERING ───────────────────────────────────────────────────
     {
         "type": "function",
         "function": {
             "name": "search_properties",
-            "description": "Search and filter properties by text, market, platform, bedrooms, or availability.",
+            "description": "Search and filter tracked properties by text, market, platform, bedrooms, or availability. Returns compact snapshot rows. Use for 'show me 2-bedroom properties in Miami', 'find available Airbnb listings in NYC', 'search for Brickell properties'.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "p_search": {"type": "string", "description": "Search text (name or uuid)"},
-                    "p_market": {"type": "string", "description": "Market region name"},
-                    "p_platform": {"type": "string", "description": "Platform (e.g. 'Airbnb')"},
+                    "p_search": {"type": "string", "description": "Search text (property name or UUID)"},
+                    "p_market": {"type": "string", "description": "Market region name (e.g. 'Miami' or 'NYC/NJ Metro')"},
+                    "p_platform": {"type": "string", "description": "Platform filter (e.g. 'airbnb' or 'vrbo')"},
                     "p_bedrooms": {"type": "integer", "description": "Number of bedrooms"},
-                    "p_available": {"type": "boolean", "description": "Availability status"},
-                    "p_limit": {"type": "integer", "default": 50, "description": "Number of results to return (max 200)"}
+                    "p_available": {"type": "boolean", "description": "true = available only, false = unavailable only"},
+                    "p_limit": {"type": "integer", "default": 20, "description": "Results to return (max 50)"}
                 },
                 "required": []
             }
@@ -119,16 +183,48 @@ REAL_ESTATE_TOOLS = [
     {
         "type": "function",
         "function": {
-            "name": "get_property_rate_changes",
-            "description": "Show historical rate changes and trailing averages for a specific property over time.",
+            "name": "get_availability_rate",
+            "description": "Get the percentage of tracked listings currently available vs unavailable within the system's 2-night check-in window, optionally filtered by market or platform. IMPORTANT: always caveat that this reflects the 2-night window only, not full-calendar occupancy.",
             "parameters": {
                 "type": "object",
                 "properties": {
-                    "property_search": {"type": "string", "description": "Property name or UUID (required)"},
-                    "days_param": {"type": "integer", "default": 14, "description": "Lookback window in days (max 90)"},
-                    "compare_window_days": {"type": "integer", "default": 1, "description": "Days to compare against for pct_change_vs_prev (max 14)"}
+                    "market": {"type": "string", "description": "Optional market filter ('Miami' or 'NYC/NJ Metro')"},
+                    "platform": {"type": "string", "description": "Optional platform filter ('airbnb' or 'vrbo')"}
                 },
-                "required": ["property_search"]
+                "required": []
+            }
+        }
+    },
+
+    # ─── GEO / LOCATION ──────────────────────────────────────────────────────
+    {
+        "type": "function",
+        "function": {
+            "name": "geocode_address",
+            "description": "Convert a user-supplied street address or general location text into latitude/longitude coordinates. Call this FIRST whenever the user gives an address rather than coordinates, BEFORE calling get_nearby_properties.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "address": {"type": "string", "description": "Street address, city, or general location text supplied by the user (US locations only)"}
+                },
+                "required": ["address"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_nearby_properties",
+            "description": "Find tracked properties within a given radius of a latitude/longitude point. Use for 'what properties are near [address]?', 'show me listings around [location]'. If the user gave an address, call geocode_address first to get coordinates.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "latitude": {"type": "number", "description": "Latitude of search center"},
+                    "longitude": {"type": "number", "description": "Longitude of search center"},
+                    "radius_km": {"type": "number", "default": 5.0, "description": "Search radius in kilometres (max 20)"},
+                    "limit": {"type": "integer", "default": 10, "description": "Max results to return (max 20)"}
+                },
+                "required": ["latitude", "longitude"]
             }
         }
     },
@@ -136,7 +232,7 @@ REAL_ESTATE_TOOLS = [
         "type": "function",
         "function": {
             "name": "get_distance_km",
-            "description": "Calculate the geographic distance in kilometers between two properties.",
+            "description": "Calculate the geographic distance in kilometres between two specific tracked properties using their UUIDs. Use for 'how far apart are property A and property B?' questions when UUIDs are known.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -146,5 +242,52 @@ REAL_ESTATE_TOOLS = [
                 "required": ["property_a_id", "property_b_id"]
             }
         }
+    },
+
+    # ─── TRACKING & SYSTEM META ───────────────────────────────────────────────
+    {
+        "type": "function",
+        "function": {
+            "name": "get_tracked_markets",
+            "description": "Fetch which markets are actively tracked, optionally filtered by platform. Use for 'what markets do you cover?', 'how many Airbnb properties in each market?' questions.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "p_platform": {"type": "string", "description": "Optional platform filter (e.g. 'Airbnb' or 'Vrbo')"}
+                },
+                "required": []
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_recently_changed_tracking",
+            "description": "Get properties recently added to or removed from active tracking. Use for 'has anything changed in what you track recently?', 'what properties were added or dropped?'. Note: removal detection is approximate based on system activity records.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "days": {"type": "integer", "default": 30, "description": "Lookback window in days"}
+                },
+                "required": []
+            }
+        }
+    },
+
+    # ─── DATA EXPORT ─────────────────────────────────────────────────────────
+    {
+        "type": "function",
+        "function": {
+            "name": "generate_data_export",
+            "description": "Generate a downloadable report or data export (CSV or Markdown) based on data the user wants to save. Use when the user says 'export', 'download', 'give me a file', or 'save this as a report'. The content should be the full formatted data from prior tool results.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "format": {"type": "string", "description": "Export format: 'csv' or 'md'"},
+                    "content": {"type": "string", "description": "Full text content or CSV data to export"}
+                },
+                "required": ["format", "content"]
+            }
+        }
     }
 ]