Codebot / repochat /models.py
rcs3's picture
model upgrading
01f4852 verified
Raw History Blame Contribute Delete
918 Bytes
import os
import streamlit as st
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_groq import ChatGroq
def hf_embeddings():
"""Lightweight sentence-transformer embeddings (runs on CPU, no GPU needed)."""
return HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2",
model_kwargs={"device": "cpu"},
)
def groq_llm():
"""
Fast, free Llama-3 inference via Groq.
Requires GROQ_API_KEY set as a HuggingFace Space secret.
"""
api_key = os.environ.get("GROQ_API_KEY", "")
if not api_key:
st.error(
"⚠️ GROQ_API_KEY secret is not set. "
"Add it in your Space → Settings → Repository secrets."
)
st.stop()
return ChatGroq(
model="llama-3.1-8b-instant",
api_key=api_key,
temperature=0.2,
max_tokens=1024,
streaming=False,
)