Rachit-Mangawa commited on
Commit
f673c95
·
1 Parent(s): a219be1

perf: optimize local inference speed and increase internal timeout to 28s

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -49,11 +49,11 @@ async def call_llm_local(prompt: str, system: str = "") -> Optional[Dict]:
49
  "prompt": f"System: {system}\nStrict Grounding. No Fabrication. Context: {prompt}\nUser: Return JSON message.",
50
  "stream": False,
51
  "format": "json",
52
- "options": {"num_predict": 120, "temperature": 0.1}
53
  }
54
  async with httpx.AsyncClient() as client:
55
  try:
56
- response = await client.post("http://127.0.0.1:11434/api/generate", json=payload, timeout=25.0)
57
  if response.status_code == 200:
58
  return json.loads(response.json().get('response', '{}'))
59
  except Exception as e:
 
49
  "prompt": f"System: {system}\nStrict Grounding. No Fabrication. Context: {prompt}\nUser: Return JSON message.",
50
  "stream": False,
51
  "format": "json",
52
+ "options": {"num_predict": 100, "temperature": 0.1}
53
  }
54
  async with httpx.AsyncClient() as client:
55
  try:
56
+ response = await client.post("http://127.0.0.1:11434/api/generate", json=payload, timeout=28.0)
57
  if response.status_code == 200:
58
  return json.loads(response.json().get('response', '{}'))
59
  except Exception as e: