File size: 1,556 Bytes
e64b456 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | from groq import Groq
import json
import os
import re
# Use environment variable for API key (never hardcode secrets!)
api_key = os.getenv("GROQ_API_KEY")
if not api_key:
raise ValueError("GROQ_API_KEY environment variable not set")
client = Groq(api_key=api_key)
prompt = """
Generate 50 additional domain-specific evaluation questions and answers
for GraphRAG, LLMs, RAG, Knowledge Graphs, Transformers, Embeddings.
Return ONLY valid JSON:
[
{
"question":"...",
"ground_truth":"..."
}
]
"""
response = client.chat.completions.create(
model="llama-3.3-70b-versatile",
messages=[{"role":"user","content":prompt}],
temperature=0.7
)
content = response.choices[0].message.content
# Remove ```json and ```
content = re.sub(r"```json|```", "", content).strip()
# Extract only the JSON array
match = re.search(r"\[.*\]", content, re.DOTALL)
if not match:
print("LLM did not return valid JSON")
print(content)
exit()
new_questions = json.loads(match.group())
existing = []
if os.path.exists("data/eval_queries.json"):
with open(
"data/eval_queries.json",
"r",
encoding="utf-8"
) as f:
existing = json.load(f)
existing_questions = {
q["question"] for q in existing
}
for item in new_questions:
if item["question"] not in existing_questions:
existing.append(item)
with open(
"data/eval_queries.json",
"w",
encoding="utf-8"
) as f:
json.dump(existing, f, indent=2)
print(
f"Added {len(existing)-len(existing_questions)} new questions"
) |