data/eval_queries_16.json

[
  {
    "question": "What is the relationship between transformer architecture and attention mechanisms in modern LLMs?",
    "ground_truth": "Transformer architecture relies fundamentally on attention mechanisms, specifically self-attention, which allows the model to weigh the importance of different tokens relative to each other. Multi-head attention enables the model to capture multiple types of relationships simultaneously across different representation subspaces."
  },
  {
    "question": "How does BERT differ from GPT in terms of training objectives and use cases?",
    "ground_truth": "BERT uses masked language modeling and next sentence prediction for bidirectional context understanding, making it ideal for classification and extraction tasks. GPT uses autoregressive next-token prediction for unidirectional generation, making it better suited for text generation tasks."
  },
  {
    "question": "What are the main causes of hallucination in large language models?",
    "ground_truth": "LLM hallucinations stem from training data gaps, overconfident probability distributions, inability to distinguish knowledge boundaries, reward hacking during RLHF, and lack of grounding to external verified knowledge sources."
  },
  {
    "question": "How does retrieval-augmented generation improve factual accuracy compared to parametric knowledge?",
    "ground_truth": "RAG improves factual accuracy by grounding model responses in retrieved external documents, reducing reliance on potentially outdated or incorrect parametric knowledge stored in model weights, and providing verifiable source attribution."
  },
  {
    "question": "What is the role of knowledge graphs in reducing LLM token consumption?",
    "ground_truth": "Knowledge graphs reduce token consumption by organizing information as structured entity-relationship triples, enabling precise retrieval of only relevant facts rather than large document chunks, resulting in compact structured prompts."
  },
  {
    "question": "How does multi-hop reasoning work in graph-based retrieval systems?",
    "ground_truth": "Multi-hop reasoning in graph retrieval follows relationship edges across multiple nodes to connect distant concepts. Starting from seed entities in the query, the system traverses the graph hop by hop, accumulating related entities and relationships that collectively provide richer context than any single document chunk."
  },
  {
    "question": "What is the difference between vector similarity search and graph traversal for RAG?",
    "ground_truth": "Vector similarity search retrieves documents by semantic closeness in embedding space, which can miss structural relationships. Graph traversal follows explicit entity-relationship edges, enabling precise multi-hop reasoning and returning only structurally connected facts relevant to the query."
  },
  {
    "question": "How does TigerGraph's GSQL support complex graph analytics?",
    "ground_truth": "GSQL is TigerGraph's graph query language that supports multi-hop traversals, pattern matching, accumulator-based aggregations, and parallel computation across the graph, enabling complex analytical queries that would be expensive in traditional databases."
  },
  {
    "question": "What metrics are used to evaluate answer quality in RAG systems?",
    "ground_truth": "RAG answer quality is evaluated using BERTScore for semantic similarity, LLM-as-a-Judge for correctness and completeness, ROUGE scores for lexical overlap, faithfulness metrics to check grounding in retrieved context, and human evaluation for overall quality."
  },
  {
    "question": "How does chunking strategy affect RAG performance?",
    "ground_truth": "Chunking strategy affects RAG by determining the granularity of retrieved context. Smaller chunks improve precision but may lose surrounding context. Larger chunks provide more context but increase token cost and noise. Overlap between chunks helps preserve cross-boundary information. Semantic chunking by sentence or paragraph boundaries outperforms fixed-size chunking."
  },
  {
    "question": "What are embedding models and how do they enable semantic search?",
    "ground_truth": "Embedding models convert text into dense numerical vectors in a high-dimensional space where semantically similar texts are placed close together. Semantic search uses these vectors with similarity metrics like cosine similarity to retrieve documents whose meaning matches the query, even without exact keyword overlap."
  },
  {
    "question": "What is the context window limitation in LLMs and how does RAG address it?",
    "ground_truth": "LLMs have fixed context windows limiting how much text they can process at once. RAG addresses this by selectively retrieving only the most relevant document segments, keeping the context within window limits while still drawing on large external knowledge bases."
  },
  {
    "question": "How does FAISS enable efficient similarity search at scale?",
    "ground_truth": "FAISS enables efficient similarity search by using approximate nearest neighbor algorithms like IVF (inverted file index) and HNSW (hierarchical navigable small world graphs) that trade small accuracy losses for massive speed gains, allowing billion-scale vector search in milliseconds."
  },
  {
    "question": "What is the significance of BERTScore F1 in evaluating generated text?",
    "ground_truth": "BERTScore F1 measures semantic similarity between generated and reference text using contextual BERT embeddings, capturing meaning-level overlap rather than just surface lexical matches. An F1 above 0.88 indicates strong semantic equivalence between the generated answer and the ground truth."
  },
  {
    "question": "How do knowledge graph entity types improve retrieval precision?",
    "ground_truth": "Entity type classification (Person, Organization, Concept, Location, Event) allows the retrieval system to filter and prioritize entities relevant to the query type, reducing irrelevant traversal paths and improving the signal-to-noise ratio of retrieved context."
  },
  {
    "question": "What is the LLM-as-a-Judge evaluation methodology?",
    "ground_truth": "LLM-as-a-Judge uses a powerful LLM to evaluate generated answers on dimensions like correctness, completeness, relevance, and clarity by comparing them to reference answers or grading them independently. It provides scalable quality assessment without requiring large human annotation teams."
  }
]