Spaces:
Running
Running
Rachit-Mangawa commited on
Commit ·
f673c95
1
Parent(s): a219be1
perf: optimize local inference speed and increase internal timeout to 28s
Browse files
app.py
CHANGED
|
@@ -49,11 +49,11 @@ async def call_llm_local(prompt: str, system: str = "") -> Optional[Dict]:
|
|
| 49 |
"prompt": f"System: {system}\nStrict Grounding. No Fabrication. Context: {prompt}\nUser: Return JSON message.",
|
| 50 |
"stream": False,
|
| 51 |
"format": "json",
|
| 52 |
-
"options": {"num_predict":
|
| 53 |
}
|
| 54 |
async with httpx.AsyncClient() as client:
|
| 55 |
try:
|
| 56 |
-
response = await client.post("http://127.0.0.1:11434/api/generate", json=payload, timeout=
|
| 57 |
if response.status_code == 200:
|
| 58 |
return json.loads(response.json().get('response', '{}'))
|
| 59 |
except Exception as e:
|
|
|
|
| 49 |
"prompt": f"System: {system}\nStrict Grounding. No Fabrication. Context: {prompt}\nUser: Return JSON message.",
|
| 50 |
"stream": False,
|
| 51 |
"format": "json",
|
| 52 |
+
"options": {"num_predict": 100, "temperature": 0.1}
|
| 53 |
}
|
| 54 |
async with httpx.AsyncClient() as client:
|
| 55 |
try:
|
| 56 |
+
response = await client.post("http://127.0.0.1:11434/api/generate", json=payload, timeout=28.0)
|
| 57 |
if response.status_code == 200:
|
| 58 |
return json.loads(response.json().get('response', '{}'))
|
| 59 |
except Exception as e:
|