llm-app / app.py
Menixis's picture
Upload app.py
7ddccf5 verified
Raw History Blame Contribute Delete
8.09 kB
import json
import os
import re
import gradio as gr
from fastapi import FastAPI, Body
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
MODEL_REPO = os.environ.get("MODEL_REPO", "unsloth/gemma-4-E4B-it-GGUF")
MODEL_FILE = os.environ.get("MODEL_FILE", "gemma-4-E4B-it-Q3_K_M.gguf")
N_CTX = int(os.environ.get("N_CTX", "4096"))
N_THREADS = int(os.environ.get("N_THREADS", "4"))
HF_TOKEN = os.environ.get("HF_TOKEN")
print(f"Downloading {MODEL_FILE} from {MODEL_REPO}...")
model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE, token=HF_TOKEN)
print(f"Model downloaded to {model_path}")
print("Loading model...")
llm = Llama(
model_path=model_path,
n_ctx=N_CTX,
n_threads=N_THREADS,
verbose=True,
)
print("Model loaded successfully!")
CLASSIFICATION_SYSTEM_PROMPT = """You are an email classification agent.
Your job is to review an email's subject and body and determine whether it belongs to the same category as these example emails.
Target Email Type
The target type is:
Procurement / tender process notification emails related to clarification meetings, attendance summaries, sign-in records, or attached participant lists.
These emails are typically internal work emails used to:
- notify recipients about summary results of vendor participation
- report who attended / did not attend a clarification meeting
- send sign-in documents
- send or reference attached attendance lists
- record procurement activity related to PR numbers / procurement reference numbers
Decision Rule
Classify as true if the email is the same type.
Classify as false if the email is not the same type.
Classify as true when the email is mainly about one or more of the following:
- clarification meeting attendance summary
- list of attendees or non-attendees
- sign-in sheet or attendance document
- attached participant list for a procurement/tender clarification event
- procurement activity update directly related to clarification attendance records
Classify as false when the email is mainly about other topics, such as:
- PR/PO approval
- payment or invoice follow-up
- contract award
- technical evaluation
- general meeting invitation
- unrelated procurement update
- any other email that does not focus on clarification attendance or related records
Important Rules
- Use both the subject and the body.
- Focus on the main purpose of the email, not only keywords.
- A PR number alone does not make it the same type.
- If the email is short but clearly refers to an attached attendance list or sign-in document for a procurement clarification event, classify it as true.
- If the email is procurement-related but about another step in the process, classify it as false.
Output Format
Return only JSON in this format:
{"is_same_type": true}
"""
def _extract_bool(text: str) -> bool:
"""Pull is_same_type out of the model's response, surviving extra prose."""
if text is None:
return False
m = re.search(r"\{[^{}]*\}", text, flags=re.DOTALL)
if m:
try:
obj = json.loads(m.group(0))
return bool(obj.get("is_same_type", False))
except json.JSONDecodeError:
pass
lowered = text.lower()
if re.search(r'"is_same_type"\s*:\s*true', lowered):
return True
if re.search(r'"is_same_type"\s*:\s*false', lowered):
return False
return False
def chat(message: str, history: list[dict]) -> str:
messages = []
for msg in history:
messages.append({"role": msg["role"], "content": msg["content"]})
messages.append({"role": "user", "content": message})
response = llm.create_chat_completion(
messages=messages,
max_tokens=1024,
temperature=0.7,
top_p=0.9,
)
return response["choices"][0]["message"]["content"]
def api_generate(prompt: str, system_prompt: str = "", max_tokens: int = 1024, temperature: float = 0.7) -> str:
"""Generic API endpoint. Kept for backward compatibility."""
messages = []
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
messages.append({"role": "user", "content": prompt})
response = llm.create_chat_completion(
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
top_p=0.9,
)
return response["choices"][0]["message"]["content"]
def classify_email(email_text: str) -> bool:
"""Classify a single email against the baked-in target type.
Power Automate sends only the email body (subject + body concatenated).
Returns a plain boolean — no JSON re-parse required on the caller side.
"""
if not email_text or not email_text.strip():
return False
messages = [
{"role": "system", "content": CLASSIFICATION_SYSTEM_PROMPT},
{"role": "user", "content": email_text},
]
response = llm.create_chat_completion(
messages=messages,
max_tokens=64,
temperature=0.1,
top_p=0.9,
)
raw = response["choices"][0]["message"]["content"]
return _extract_bool(raw)
with gr.Blocks(title="LLM App") as demo:
gr.Markdown("# LLM App")
gr.Markdown(
"Chat with the model, use the generic API endpoint, "
"or call the dedicated email-classification endpoint for Power Automate."
)
chatbot = gr.ChatInterface(fn=chat, type="messages")
gr.Markdown("---")
gr.Markdown("## Email Classification Endpoint (for Power Automate)")
gr.Markdown(
"Send a POST request to this Space's `/api/classify_email` endpoint. "
"The classification system prompt is baked into the Space, so the payload only needs the email text.\n\n"
"**Example payload:**\n"
"```json\n"
'{"data": ["Subject: <subject>\\n\\nBody:\\n<body>"]}\n'
"```\n\n"
"**Response:**\n"
"```json\n"
'{"data": [true]}\n'
"```\n"
)
classify_interface = gr.Interface(
fn=classify_email,
inputs=[
gr.Textbox(
label="Email (subject + body)",
placeholder="Subject: ...\n\nBody:\n...",
lines=10,
),
],
outputs=gr.Checkbox(label="is_same_type"),
api_name="classify_email",
allow_flagging="never",
)
gr.Markdown("---")
gr.Markdown("## Generic Generation Endpoint")
gr.Markdown(
"Send a POST request to this Space's `/api/api_generate` endpoint.\n\n"
"**Example payload:**\n"
"```json\n"
'{"data": ["Your prompt here", "Optional system prompt", 1024, 0.7]}\n'
"```"
)
api_interface = gr.Interface(
fn=api_generate,
inputs=[
gr.Textbox(label="Prompt", placeholder="Enter your prompt..."),
gr.Textbox(label="System Prompt (optional)", placeholder="You are a helpful assistant..."),
gr.Slider(minimum=64, maximum=4096, value=1024, step=64, label="Max Tokens"),
gr.Slider(minimum=0.0, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
],
outputs=gr.Textbox(label="Response"),
api_name="api_generate",
allow_flagging="never",
)
fastapi_app = FastAPI()
@fastapi_app.post("/api/classify")
def classify_email_endpoint(payload: dict = Body(...)):
"""Synchronous POST endpoint for Power Automate and other no-poll callers.
Send: {"email": "Subject: ...\\nBody: ..."} (or {"text": "..."} / {"data": ["..."]})
Get back: {"is_same_type": true|false}
"""
if isinstance(payload, dict):
email_text = (
payload.get("email")
or payload.get("text")
or (payload.get("data", [""]) or [""])[0]
or ""
)
else:
email_text = ""
return {"is_same_type": classify_email(str(email_text))}
@fastapi_app.get("/api/health")
def health():
return {"status": "ok"}
if __name__ == "__main__":
import uvicorn
app = gr.mount_gradio_app(fastapi_app, demo, path="/")
uvicorn.run(app, host="0.0.0.0", port=7860)