Spaces:
Sleeping
Sleeping
testing and bug fix
Browse files- .env.example +5 -1
- app/core/config.py +5 -2
- app/rag/llm.py +43 -5
- app/rag/router.py +18 -10
- scripts/test_chat.py +2 -0
.env.example
CHANGED
|
@@ -6,8 +6,12 @@ DATABASE_URL=postgresql+psycopg://creator:creator@localhost:5433/creator_rag
|
|
| 6 |
# Swap either model name to change providers without touching code (NFR-5).
|
| 7 |
OPENROUTER_API_KEY=sk-or-...
|
| 8 |
OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
|
| 9 |
-
|
|
|
|
|
|
|
| 10 |
SYNTHESIS_MODEL=anthropic/claude-sonnet-4.6
|
|
|
|
|
|
|
| 11 |
|
| 12 |
# ---- Instagram metadata provider ----
|
| 13 |
# instaloader (free, local, demo-grade) | apify (managed, paid, production)
|
|
|
|
| 6 |
# Swap either model name to change providers without touching code (NFR-5).
|
| 7 |
OPENROUTER_API_KEY=sk-or-...
|
| 8 |
OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
|
| 9 |
+
# Cheap, fast, reliable at one-word intent classification. A smaller open model
|
| 10 |
+
# (e.g. meta-llama/llama-3.1-8b-instruct) is cheaper but misroutes more often.
|
| 11 |
+
ROUTER_MODEL=anthropic/claude-3.5-haiku
|
| 12 |
SYNTHESIS_MODEL=anthropic/claude-sonnet-4.6
|
| 13 |
+
# Cap synthesis output tokens (bounds cost; avoids OpenRouter credit pre-reservation).
|
| 14 |
+
SYNTHESIS_MAX_TOKENS=1024
|
| 15 |
|
| 16 |
# ---- Instagram metadata provider ----
|
| 17 |
# instaloader (free, local, demo-grade) | apify (managed, paid, production)
|
app/core/config.py
CHANGED
|
@@ -24,8 +24,11 @@ class Settings(BaseSettings):
|
|
| 24 |
# LLM via OpenRouter (two tiers, swappable by model name)
|
| 25 |
openrouter_api_key: str = ""
|
| 26 |
openrouter_base_url: str = "https://openrouter.ai/api/v1"
|
| 27 |
-
router_model: str = "
|
| 28 |
-
synthesis_model: str = "anthropic/claude-
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
# Instagram metadata provider: "instaloader" | "apify"
|
| 31 |
ig_provider: str = "instaloader"
|
|
|
|
| 24 |
# LLM via OpenRouter (two tiers, swappable by model name)
|
| 25 |
openrouter_api_key: str = ""
|
| 26 |
openrouter_base_url: str = "https://openrouter.ai/api/v1"
|
| 27 |
+
router_model: str = "anthropic/claude-3.5-haiku"
|
| 28 |
+
synthesis_model: str = "anthropic/claude-sonnet-4.6"
|
| 29 |
+
# Cap synthesis output: bounds per-query cost and avoids OpenRouter
|
| 30 |
+
# pre-reserving credit for the model's full context window.
|
| 31 |
+
synthesis_max_tokens: int = 1024
|
| 32 |
|
| 33 |
# Instagram metadata provider: "instaloader" | "apify"
|
| 34 |
ig_provider: str = "instaloader"
|
app/rag/llm.py
CHANGED
|
@@ -8,29 +8,67 @@ from __future__ import annotations
|
|
| 8 |
|
| 9 |
from functools import lru_cache
|
| 10 |
|
|
|
|
| 11 |
from langchain_openai import ChatOpenAI
|
| 12 |
|
| 13 |
from app.core.config import get_settings
|
| 14 |
|
| 15 |
|
| 16 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
s = get_settings()
|
|
|
|
| 18 |
return ChatOpenAI(
|
| 19 |
model=model,
|
| 20 |
temperature=temperature,
|
| 21 |
streaming=streaming,
|
|
|
|
| 22 |
api_key=s.openrouter_api_key,
|
| 23 |
base_url=s.openrouter_base_url,
|
|
|
|
|
|
|
| 24 |
)
|
| 25 |
|
| 26 |
|
| 27 |
@lru_cache(maxsize=1)
|
| 28 |
def get_router_llm() -> ChatOpenAI:
|
| 29 |
-
"""Cheap model for
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
@lru_cache(maxsize=1)
|
| 34 |
def get_synthesis_llm() -> ChatOpenAI:
|
| 35 |
-
"""Flagship model for the streamed, cited final answer.
|
| 36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
from functools import lru_cache
|
| 10 |
|
| 11 |
+
import httpx
|
| 12 |
from langchain_openai import ChatOpenAI
|
| 13 |
|
| 14 |
from app.core.config import get_settings
|
| 15 |
|
| 16 |
|
| 17 |
+
@lru_cache(maxsize=1)
|
| 18 |
+
def _http_clients() -> tuple[httpx.Client, httpx.AsyncClient]:
|
| 19 |
+
"""Sync + async HTTP clients that force IPv4.
|
| 20 |
+
|
| 21 |
+
Binding the local address to 0.0.0.0 keeps the OpenAI SDK on IPv4. Without
|
| 22 |
+
this, the first TCP connect tries an unreachable IPv6 route and blocks for
|
| 23 |
+
the full ~75s connect timeout before falling back (curl avoids this via
|
| 24 |
+
Happy Eyeballs). Reused across calls so the connection pool stays warm.
|
| 25 |
+
"""
|
| 26 |
+
return (
|
| 27 |
+
httpx.Client(transport=httpx.HTTPTransport(local_address="0.0.0.0")),
|
| 28 |
+
httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(local_address="0.0.0.0")),
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _make(
|
| 33 |
+
model: str, *, temperature: float, streaming: bool, max_tokens: int | None = None
|
| 34 |
+
) -> ChatOpenAI:
|
| 35 |
s = get_settings()
|
| 36 |
+
sync_client, async_client = _http_clients()
|
| 37 |
return ChatOpenAI(
|
| 38 |
model=model,
|
| 39 |
temperature=temperature,
|
| 40 |
streaming=streaming,
|
| 41 |
+
max_tokens=max_tokens,
|
| 42 |
api_key=s.openrouter_api_key,
|
| 43 |
base_url=s.openrouter_base_url,
|
| 44 |
+
http_client=sync_client,
|
| 45 |
+
http_async_client=async_client,
|
| 46 |
)
|
| 47 |
|
| 48 |
|
| 49 |
@lru_cache(maxsize=1)
|
| 50 |
def get_router_llm() -> ChatOpenAI:
|
| 51 |
+
"""Cheap model for intent classification. Deterministic, one-word output.
|
| 52 |
+
|
| 53 |
+
Tight max_tokens: the reply is a single word, and an uncapped default makes
|
| 54 |
+
OpenRouter pre-reserve credit for the model's full output window.
|
| 55 |
+
"""
|
| 56 |
+
return _make(
|
| 57 |
+
get_settings().router_model, temperature=0.0, streaming=False, max_tokens=16
|
| 58 |
+
)
|
| 59 |
|
| 60 |
|
| 61 |
@lru_cache(maxsize=1)
|
| 62 |
def get_synthesis_llm() -> ChatOpenAI:
|
| 63 |
+
"""Flagship model for the streamed, cited final answer.
|
| 64 |
+
|
| 65 |
+
Cap output tokens: keeps answers tight and bounds per-query cost. Without a
|
| 66 |
+
cap, OpenRouter pre-reserves credit for the model's full context (~65k),
|
| 67 |
+
which can 402 on low-balance accounts before generating anything.
|
| 68 |
+
"""
|
| 69 |
+
return _make(
|
| 70 |
+
get_settings().synthesis_model,
|
| 71 |
+
temperature=0.3,
|
| 72 |
+
streaming=True,
|
| 73 |
+
max_tokens=get_settings().synthesis_max_tokens,
|
| 74 |
+
)
|
app/rag/router.py
CHANGED
|
@@ -6,14 +6,14 @@ retrieved; qualitative ones are real retrieval; hybrid needs both (README).
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
-
from typing import Literal
|
| 10 |
|
| 11 |
from langchain_core.messages import HumanMessage, SystemMessage
|
| 12 |
-
from pydantic import BaseModel, Field
|
| 13 |
|
| 14 |
from app.rag.llm import get_router_llm
|
| 15 |
|
| 16 |
Intent = Literal["quantitative", "qualitative", "hybrid"]
|
|
|
|
| 17 |
|
| 18 |
_SYSTEM = """You classify a user's question about a comparison of two short-form \
|
| 19 |
videos (A = YouTube, B = Instagram) into exactly one intent:
|
|
@@ -26,17 +26,25 @@ suggestions for improvement. Needs retrieval over transcript text.
|
|
| 26 |
- "hybrid": needs BOTH a number and content reasoning (e.g. "why did A get more \
|
| 27 |
engagement than B?").
|
| 28 |
|
| 29 |
-
Use the chat history to resolve follow-ups.
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
class _Classification(BaseModel):
|
| 33 |
-
intent: Intent = Field(description="One of quantitative, qualitative, hybrid")
|
| 34 |
|
| 35 |
|
| 36 |
def classify(question: str, history: str = "") -> Intent:
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
user = question if not history else f"Chat history:\n{history}\n\nQuestion: {question}"
|
| 39 |
-
|
| 40 |
[SystemMessage(content=_SYSTEM), HumanMessage(content=user)]
|
| 41 |
)
|
| 42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
+
from typing import Literal, get_args
|
| 10 |
|
| 11 |
from langchain_core.messages import HumanMessage, SystemMessage
|
|
|
|
| 12 |
|
| 13 |
from app.rag.llm import get_router_llm
|
| 14 |
|
| 15 |
Intent = Literal["quantitative", "qualitative", "hybrid"]
|
| 16 |
+
_INTENTS = get_args(Intent)
|
| 17 |
|
| 18 |
_SYSTEM = """You classify a user's question about a comparison of two short-form \
|
| 19 |
videos (A = YouTube, B = Instagram) into exactly one intent:
|
|
|
|
| 26 |
- "hybrid": needs BOTH a number and content reasoning (e.g. "why did A get more \
|
| 27 |
engagement than B?").
|
| 28 |
|
| 29 |
+
Use the chat history to resolve follow-ups. Reply with ONE word only — exactly \
|
| 30 |
+
one of: quantitative, qualitative, hybrid. No punctuation, no explanation."""
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def classify(question: str, history: str = "") -> Intent:
|
| 34 |
+
"""Classify intent with a plain one-word completion.
|
| 35 |
+
|
| 36 |
+
Deliberately avoids `.with_structured_output()` / tool-calling: OpenRouter
|
| 37 |
+
may route the cheap model to a provider whose function-calling fails (e.g.
|
| 38 |
+
Groq returning "Failed to call a function"). A parsed word is provider-
|
| 39 |
+
agnostic. Falls back to "hybrid" — the safe superset that gathers both
|
| 40 |
+
metrics and transcript evidence — if the reply is unrecognized.
|
| 41 |
+
"""
|
| 42 |
user = question if not history else f"Chat history:\n{history}\n\nQuestion: {question}"
|
| 43 |
+
reply = get_router_llm().invoke(
|
| 44 |
[SystemMessage(content=_SYSTEM), HumanMessage(content=user)]
|
| 45 |
)
|
| 46 |
+
text = (reply.content or "").strip().lower()
|
| 47 |
+
for intent in _INTENTS:
|
| 48 |
+
if intent in text:
|
| 49 |
+
return intent # type: ignore[return-value]
|
| 50 |
+
return "hybrid"
|
scripts/test_chat.py
CHANGED
|
@@ -44,6 +44,8 @@ def ask(label: str, message: str) -> None:
|
|
| 44 |
answer_parts.append(evt["content"])
|
| 45 |
sys.stdout.write(evt["content"])
|
| 46 |
sys.stdout.flush()
|
|
|
|
|
|
|
| 47 |
elif t == "done":
|
| 48 |
break
|
| 49 |
|
|
|
|
| 44 |
answer_parts.append(evt["content"])
|
| 45 |
sys.stdout.write(evt["content"])
|
| 46 |
sys.stdout.flush()
|
| 47 |
+
elif t == "error":
|
| 48 |
+
print(f"\n[SERVER ERROR] {evt['message']}")
|
| 49 |
elif t == "done":
|
| 50 |
break
|
| 51 |
|