xPushkal commited on
Commit
e90c3c4
·
1 Parent(s): 43a4b46

testing and bug fix

Browse files
.env.example CHANGED
@@ -6,8 +6,12 @@ DATABASE_URL=postgresql+psycopg://creator:creator@localhost:5433/creator_rag
6
  # Swap either model name to change providers without touching code (NFR-5).
7
  OPENROUTER_API_KEY=sk-or-...
8
  OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
9
- ROUTER_MODEL=meta-llama/llama-3.1-8b-instruct
 
 
10
  SYNTHESIS_MODEL=anthropic/claude-sonnet-4.6
 
 
11
 
12
  # ---- Instagram metadata provider ----
13
  # instaloader (free, local, demo-grade) | apify (managed, paid, production)
 
6
  # Swap either model name to change providers without touching code (NFR-5).
7
  OPENROUTER_API_KEY=sk-or-...
8
  OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
9
+ # Cheap, fast, reliable at one-word intent classification. A smaller open model
10
+ # (e.g. meta-llama/llama-3.1-8b-instruct) is cheaper but misroutes more often.
11
+ ROUTER_MODEL=anthropic/claude-3.5-haiku
12
  SYNTHESIS_MODEL=anthropic/claude-sonnet-4.6
13
+ # Cap synthesis output tokens (bounds cost; avoids OpenRouter credit pre-reservation).
14
+ SYNTHESIS_MAX_TOKENS=1024
15
 
16
  # ---- Instagram metadata provider ----
17
  # instaloader (free, local, demo-grade) | apify (managed, paid, production)
app/core/config.py CHANGED
@@ -24,8 +24,11 @@ class Settings(BaseSettings):
24
  # LLM via OpenRouter (two tiers, swappable by model name)
25
  openrouter_api_key: str = ""
26
  openrouter_base_url: str = "https://openrouter.ai/api/v1"
27
- router_model: str = "meta-llama/llama-3.1-8b-instruct"
28
- synthesis_model: str = "anthropic/claude-3.5-sonnet"
 
 
 
29
 
30
  # Instagram metadata provider: "instaloader" | "apify"
31
  ig_provider: str = "instaloader"
 
24
  # LLM via OpenRouter (two tiers, swappable by model name)
25
  openrouter_api_key: str = ""
26
  openrouter_base_url: str = "https://openrouter.ai/api/v1"
27
+ router_model: str = "anthropic/claude-3.5-haiku"
28
+ synthesis_model: str = "anthropic/claude-sonnet-4.6"
29
+ # Cap synthesis output: bounds per-query cost and avoids OpenRouter
30
+ # pre-reserving credit for the model's full context window.
31
+ synthesis_max_tokens: int = 1024
32
 
33
  # Instagram metadata provider: "instaloader" | "apify"
34
  ig_provider: str = "instaloader"
app/rag/llm.py CHANGED
@@ -8,29 +8,67 @@ from __future__ import annotations
8
 
9
  from functools import lru_cache
10
 
 
11
  from langchain_openai import ChatOpenAI
12
 
13
  from app.core.config import get_settings
14
 
15
 
16
- def _make(model: str, *, temperature: float, streaming: bool) -> ChatOpenAI:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  s = get_settings()
 
18
  return ChatOpenAI(
19
  model=model,
20
  temperature=temperature,
21
  streaming=streaming,
 
22
  api_key=s.openrouter_api_key,
23
  base_url=s.openrouter_base_url,
 
 
24
  )
25
 
26
 
27
  @lru_cache(maxsize=1)
28
  def get_router_llm() -> ChatOpenAI:
29
- """Cheap model for structured intent classification. Deterministic."""
30
- return _make(get_settings().router_model, temperature=0.0, streaming=False)
 
 
 
 
 
 
31
 
32
 
33
  @lru_cache(maxsize=1)
34
  def get_synthesis_llm() -> ChatOpenAI:
35
- """Flagship model for the streamed, cited final answer."""
36
- return _make(get_settings().synthesis_model, temperature=0.3, streaming=True)
 
 
 
 
 
 
 
 
 
 
 
8
 
9
  from functools import lru_cache
10
 
11
+ import httpx
12
  from langchain_openai import ChatOpenAI
13
 
14
  from app.core.config import get_settings
15
 
16
 
17
+ @lru_cache(maxsize=1)
18
+ def _http_clients() -> tuple[httpx.Client, httpx.AsyncClient]:
19
+ """Sync + async HTTP clients that force IPv4.
20
+
21
+ Binding the local address to 0.0.0.0 keeps the OpenAI SDK on IPv4. Without
22
+ this, the first TCP connect tries an unreachable IPv6 route and blocks for
23
+ the full ~75s connect timeout before falling back (curl avoids this via
24
+ Happy Eyeballs). Reused across calls so the connection pool stays warm.
25
+ """
26
+ return (
27
+ httpx.Client(transport=httpx.HTTPTransport(local_address="0.0.0.0")),
28
+ httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(local_address="0.0.0.0")),
29
+ )
30
+
31
+
32
+ def _make(
33
+ model: str, *, temperature: float, streaming: bool, max_tokens: int | None = None
34
+ ) -> ChatOpenAI:
35
  s = get_settings()
36
+ sync_client, async_client = _http_clients()
37
  return ChatOpenAI(
38
  model=model,
39
  temperature=temperature,
40
  streaming=streaming,
41
+ max_tokens=max_tokens,
42
  api_key=s.openrouter_api_key,
43
  base_url=s.openrouter_base_url,
44
+ http_client=sync_client,
45
+ http_async_client=async_client,
46
  )
47
 
48
 
49
  @lru_cache(maxsize=1)
50
  def get_router_llm() -> ChatOpenAI:
51
+ """Cheap model for intent classification. Deterministic, one-word output.
52
+
53
+ Tight max_tokens: the reply is a single word, and an uncapped default makes
54
+ OpenRouter pre-reserve credit for the model's full output window.
55
+ """
56
+ return _make(
57
+ get_settings().router_model, temperature=0.0, streaming=False, max_tokens=16
58
+ )
59
 
60
 
61
  @lru_cache(maxsize=1)
62
  def get_synthesis_llm() -> ChatOpenAI:
63
+ """Flagship model for the streamed, cited final answer.
64
+
65
+ Cap output tokens: keeps answers tight and bounds per-query cost. Without a
66
+ cap, OpenRouter pre-reserves credit for the model's full context (~65k),
67
+ which can 402 on low-balance accounts before generating anything.
68
+ """
69
+ return _make(
70
+ get_settings().synthesis_model,
71
+ temperature=0.3,
72
+ streaming=True,
73
+ max_tokens=get_settings().synthesis_max_tokens,
74
+ )
app/rag/router.py CHANGED
@@ -6,14 +6,14 @@ retrieved; qualitative ones are real retrieval; hybrid needs both (README).
6
  """
7
  from __future__ import annotations
8
 
9
- from typing import Literal
10
 
11
  from langchain_core.messages import HumanMessage, SystemMessage
12
- from pydantic import BaseModel, Field
13
 
14
  from app.rag.llm import get_router_llm
15
 
16
  Intent = Literal["quantitative", "qualitative", "hybrid"]
 
17
 
18
  _SYSTEM = """You classify a user's question about a comparison of two short-form \
19
  videos (A = YouTube, B = Instagram) into exactly one intent:
@@ -26,17 +26,25 @@ suggestions for improvement. Needs retrieval over transcript text.
26
  - "hybrid": needs BOTH a number and content reasoning (e.g. "why did A get more \
27
  engagement than B?").
28
 
29
- Use the chat history to resolve follow-ups. Respond with the intent only."""
30
-
31
-
32
- class _Classification(BaseModel):
33
- intent: Intent = Field(description="One of quantitative, qualitative, hybrid")
34
 
35
 
36
  def classify(question: str, history: str = "") -> Intent:
37
- llm = get_router_llm().with_structured_output(_Classification)
 
 
 
 
 
 
 
38
  user = question if not history else f"Chat history:\n{history}\n\nQuestion: {question}"
39
- result: _Classification = llm.invoke(
40
  [SystemMessage(content=_SYSTEM), HumanMessage(content=user)]
41
  )
42
- return result.intent
 
 
 
 
 
6
  """
7
  from __future__ import annotations
8
 
9
+ from typing import Literal, get_args
10
 
11
  from langchain_core.messages import HumanMessage, SystemMessage
 
12
 
13
  from app.rag.llm import get_router_llm
14
 
15
  Intent = Literal["quantitative", "qualitative", "hybrid"]
16
+ _INTENTS = get_args(Intent)
17
 
18
  _SYSTEM = """You classify a user's question about a comparison of two short-form \
19
  videos (A = YouTube, B = Instagram) into exactly one intent:
 
26
  - "hybrid": needs BOTH a number and content reasoning (e.g. "why did A get more \
27
  engagement than B?").
28
 
29
+ Use the chat history to resolve follow-ups. Reply with ONE word only — exactly \
30
+ one of: quantitative, qualitative, hybrid. No punctuation, no explanation."""
 
 
 
31
 
32
 
33
  def classify(question: str, history: str = "") -> Intent:
34
+ """Classify intent with a plain one-word completion.
35
+
36
+ Deliberately avoids `.with_structured_output()` / tool-calling: OpenRouter
37
+ may route the cheap model to a provider whose function-calling fails (e.g.
38
+ Groq returning "Failed to call a function"). A parsed word is provider-
39
+ agnostic. Falls back to "hybrid" — the safe superset that gathers both
40
+ metrics and transcript evidence — if the reply is unrecognized.
41
+ """
42
  user = question if not history else f"Chat history:\n{history}\n\nQuestion: {question}"
43
+ reply = get_router_llm().invoke(
44
  [SystemMessage(content=_SYSTEM), HumanMessage(content=user)]
45
  )
46
+ text = (reply.content or "").strip().lower()
47
+ for intent in _INTENTS:
48
+ if intent in text:
49
+ return intent # type: ignore[return-value]
50
+ return "hybrid"
scripts/test_chat.py CHANGED
@@ -44,6 +44,8 @@ def ask(label: str, message: str) -> None:
44
  answer_parts.append(evt["content"])
45
  sys.stdout.write(evt["content"])
46
  sys.stdout.flush()
 
 
47
  elif t == "done":
48
  break
49
 
 
44
  answer_parts.append(evt["content"])
45
  sys.stdout.write(evt["content"])
46
  sys.stdout.flush()
47
+ elif t == "error":
48
+ print(f"\n[SERVER ERROR] {evt['message']}")
49
  elif t == "done":
50
  break
51