rcs3 commited on
Commit
f1e0216
·
verified ·
1 Parent(s): 9f0dfcc

Update models.py

Browse files
Files changed (1) hide show
  1. repochat/models.py +29 -18
repochat/models.py CHANGED
@@ -1,23 +1,34 @@
1
- from langchain.llms import LlamaCpp
2
- from langchain.embeddings import HuggingFaceEmbeddings
3
- from langchain.callbacks.manager import CallbackManager
4
- from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 
5
 
6
  def hf_embeddings():
 
7
  return HuggingFaceEmbeddings(
8
- model_name = "sentence-transformers/all-mpnet-base-v2",
 
9
  )
10
 
11
- def code_llama():
12
- callbackmanager = CallbackManager([StreamingStdOutCallbackHandler()])
13
- llm = LlamaCpp(
14
- model_path="./models/codellama-7b.Q4_K_M.gguf",
15
- n_ctx=2048,
16
- max_tokens=200,
17
- n_gpu_layers=1,
18
- f16_kv=True,
19
- callback_manager=callbackmanager,
20
- verbose=True,
21
- use_mlock=True
22
- )
23
- return llm
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import streamlit as st
3
+ from langchain_huggingface import HuggingFaceEmbeddings
4
+ from langchain_groq import ChatGroq
5
+
6
 
7
  def hf_embeddings():
8
+ """Lightweight sentence-transformer embeddings (runs on CPU, no GPU needed)."""
9
  return HuggingFaceEmbeddings(
10
+ model_name="sentence-transformers/all-mpnet-base-v2",
11
+ model_kwargs={"device": "cpu"},
12
  )
13
 
14
+
15
+ def groq_llm():
16
+ """
17
+ Fast, free Llama-3 inference via Groq.
18
+ Requires GROQ_API_KEY set as a HuggingFace Space secret.
19
+ """
20
+ api_key = os.environ.get("GROQ_API_KEY", "")
21
+ if not api_key:
22
+ st.error(
23
+ "⚠️ GROQ_API_KEY secret is not set. "
24
+ "Add it in your Space → Settings → Repository secrets."
25
+ )
26
+ st.stop()
27
+
28
+ return ChatGroq(
29
+ model="llama3-8b-8192",
30
+ api_key=api_key,
31
+ temperature=0.2,
32
+ max_tokens=1024,
33
+ streaming=False,
34
+ )