Spaces:

amer003100
/

vectorless_RAG

Running

vectorless_RAG / evaluation /eval_results.json
AI Assistant
docs: add quantitative evaluation benchmarks (80-question single-doc test) and report
b3242a4 10 days ago
85.6 kB
{
  "overall": {
    "count": 80,
    "retrieval_recall": 0.7308,
    "context_precision": 0.3916,
    "answer_correctness": 0.8773,
    "faithfulness": 0.9065,
    "citation_accuracy": 0.3833,
    "avg_retrieval_latency_s": 0.0071,
    "avg_total_latency_s": 1.8006,
    "avg_prompt_tokens": 2939.1,
    "avg_completion_tokens": 121.7,
    "total_cost_usd": 0.073532,
    "avg_cost_per_query_usd": 0.000919
  },
  "by_category": {
    "Local": {
      "count": 20,
      "retrieval_recall": 0.875,
      "context_precision": 0.325,
      "answer_correctness": 0.8017,
      "faithfulness": 0.94,
      "citation_accuracy": 0.275,
      "avg_retrieval_latency_s": 0.006,
      "avg_total_latency_s": 1.7516,
      "avg_prompt_tokens": 2873.7,
      "avg_completion_tokens": 118.1,
      "total_cost_usd": 0.0178,
      "avg_cost_per_query_usd": 0.00089
    },
    "Global": {
      "count": 20,
      "retrieval_recall": 0.41,
      "context_precision": 0.3417,
      "answer_correctness": 0.8708,
      "faithfulness": 0.726,
      "citation_accuracy": 0.4083,
      "avg_retrieval_latency_s": 0.0074,
      "avg_total_latency_s": 1.8334,
      "avg_prompt_tokens": 2903.7,
      "avg_completion_tokens": 124.1,
      "total_cost_usd": 0.018141,
      "avg_cost_per_query_usd": 0.000907
    },
    "Multi-hop": {
      "count": 20,
      "retrieval_recall": 0.6833,
      "context_precision": 0.475,
      "answer_correctness": 0.9833,
      "faithfulness": 0.96,
      "citation_accuracy": 0.425,
      "avg_retrieval_latency_s": 0.0088,
      "avg_total_latency_s": 1.8741,
      "avg_prompt_tokens": 3122.1,
      "avg_completion_tokens": 127.0,
      "total_cost_usd": 0.019932,
      "avg_cost_per_query_usd": 0.000997
    },
    "Summary": {
      "count": 10,
      "retrieval_recall": 0.91,
      "context_precision": 0.5,
      "answer_correctness": 0.9067,
      "faithfulness": 1.0,
      "citation_accuracy": 0.4833,
      "avg_retrieval_latency_s": 0.0064,
      "avg_total_latency_s": 2.0701,
      "avg_prompt_tokens": 2846.6,
      "avg_completion_tokens": 142.0,
      "total_cost_usd": 0.008985,
      "avg_cost_per_query_usd": 0.000898
    },
    "Numerical/Factual": {
      "count": 10,
      "retrieval_recall": 1.0,
      "context_precision": 0.35,
      "answer_correctness": 0.8,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3667,
      "avg_retrieval_latency_s": 0.0059,
      "avg_total_latency_s": 1.4163,
      "avg_prompt_tokens": 2867.3,
      "avg_completion_tokens": 92.9,
      "total_cost_usd": 0.008674,
      "avg_cost_per_query_usd": 0.000867
    }
  },
  "detailed_results": [
    {
      "id": "LQ_01",
      "category": "Local",
      "question": "What are the four distinct categories of Knowledge Graphs categorized by knowledge type in Section 2?",
      "retrieved_nodes": [
        "0000",
        "0002",
        "0006"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0072,
      "total_latency_s": 2.1372,
      "prompt_tokens": 3284,
      "completion_tokens": 147,
      "cost_usd": 0.001077,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [2 BACKGROUND (Page 2)], [4.2 KG-enhanced LLM Inference (Page 9)]: In Section 2, Knowledge Graphs are categorized into four types based on knowledge: 1) Encyclopedia Knowledge Graphs (e.g., Wikidata, DBpedia, YAGO, Freebase), 2) Commonsense Knowledge Graphs (e.g., ConceptNet, ATOMIC), 3) Domain-specific Knowledge Graphs (e.g., UMLS, SNOMED CT), and 4) Multimodal Knowledge Graphs (e.g., IMGpedia, MMKG)."
    },
    {
      "id": "LQ_02",
      "category": "Local",
      "question": "How does ERNIE (Zhang et al.) integrate entity embeddings into pre-trained language models in Section 4.1?",
      "retrieved_nodes": [
        "0005",
        "0000",
        "0004"
      ],
      "ground_truth_nodes": [
        "0005"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.25,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0069,
      "total_latency_s": 1.4302,
      "prompt_tokens": 2867,
      "completion_tokens": 94,
      "cost_usd": 0.000868,
      "answer": "Based on [4.1 KG-enhanced LLM Pre-training (Page 7)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [4 KG-ENHANCED LLMs (Page 7)], partial evidence found (multi-layer fusion encoder / mutually attentive). Full evidence requires additional sections."
    },
    {
      "id": "LQ_03",
      "category": "Local",
      "question": "What is the difference between PaE (Pre-trained as Encoders) and PaD (Pre-trained as Decoders) in Knowledge Graph Completion (Section 5.2)?",
      "retrieved_nodes": [
        "0011",
        "0007",
        "0000"
      ],
      "ground_truth_nodes": [
        "0011"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0057,
      "total_latency_s": 2.3124,
      "prompt_tokens": 3137,
      "completion_tokens": 160,
      "cost_usd": 0.001029,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: In PaE (Pre-trained as Encoders), LLMs serve as representation encoders that produce entity/relation embeddings, which are scored using prediction heads or scoring functions. In PaD (Pre-trained as Decoders), LLMs generate target missing entity names directly in an auto-regressive generative sequence-to-sequence manner via prompting."
    },
    {
      "id": "LQ_04",
      "category": "Local",
      "question": "What are the three main steps of KG-enhanced LLM inference methods described in Section 4.2?",
      "retrieved_nodes": [
        "0006",
        "0007",
        "0004"
      ],
      "ground_truth_nodes": [
        "0006"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0065,
      "total_latency_s": 2.2031,
      "prompt_tokens": 2829,
      "completion_tokens": 152,
      "cost_usd": 0.000899,
      "answer": "Based on [4.2 KG-enhanced LLM Inference (Page 9)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4 KG-ENHANCED LLMs (Page 7)]: The three main steps of KG-enhanced LLM inference are: 1) Knowledge Retrieval (identifying relevant entities/triples via entity linking or subgraph search), 2) Knowledge Representation (converting structured triples into text, graphs, or linear prompts), and 3) Knowledge Fusion (incorporating knowledge into the LLM during generation or attention)."
    },
    {
      "id": "LQ_05",
      "category": "Local",
      "question": "According to Appendix A, what are the primary advantages (pros) and disadvantages (cons) of Knowledge Graphs?",
      "retrieved_nodes": [
        "0000",
        "0020"
      ],
      "ground_truth_nodes": [
        "0020"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0062,
      "total_latency_s": 1.9628,
      "prompt_tokens": 2646,
      "completion_tokens": 134,
      "cost_usd": 0.000812,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [APPENDIX A (Page 27)]: KG pros include: Structural Knowledge (triples), High Accuracy (expert curated), Decisiveness, Interpretability (symbolic reasoning), Domain-specific Knowledge, and Evolving Knowledge. KG cons include: Incompleteness (hard to maintain), Lacking Language Understanding (ignores text nuances), and Difficulty handling Unseen Facts."
    },
    {
      "id": "LQ_06",
      "category": "Local",
      "question": "How does K-BERT incorporate knowledge graphs without retraining the base language model in Section 4.2?",
      "retrieved_nodes": [
        "0000",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0006"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 0.2,
      "faithfulness": 0.4,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 1.3122,
      "prompt_tokens": 2459,
      "completion_tokens": 85,
      "cost_usd": 0.000698,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)], partial evidence found (tree-structured sentence). Full evidence requires additional sections."
    },
    {
      "id": "LQ_07",
      "category": "Local",
      "question": "What are the two primary approaches for Knowledge Graph Embedding (KGE) using LLMs in Section 5.1?",
      "retrieved_nodes": [
        "0010",
        "0011",
        "0009"
      ],
      "ground_truth_nodes": [
        "0010"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.8993,
      "prompt_tokens": 3010,
      "completion_tokens": 129,
      "cost_usd": 0.000953,
      "answer": "Based on [5.1 LLM-augmented KG Embedding (Page 11)], [5.2 LLM-augmented KG Completion (Page 12)], [5 LLM-AUGMENTED KGs (Page 11)]: Section 5.1 classifies LLM-augmented KGE into: 1) LLM as Entity/Relation Feature Encoders (extracting contextual text representations of entity descriptions), and 2) Joint Structural and Textual Embedding (combining structural graph losses like TransE with language modeling objectives)."
    },
    {
      "id": "LQ_08",
      "category": "Local",
      "question": "What is the core concept of 'Synergized LLMs + KGs' as defined in Section 6?",
      "retrieved_nodes": [
        "0015",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0015"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0049,
      "total_latency_s": 1.7615,
      "prompt_tokens": 2579,
      "completion_tokens": 119,
      "cost_usd": 0.000773,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: Synergized LLMs + KGs refers to a bidirectional, unified framework where LLMs and KGs play mutually beneficial and reciprocal roles in representation and reasoning, combining the perceptual and linguistic capabilities of LLMs with the structured factual rigor of KGs."
    },
    {
      "id": "LQ_09",
      "category": "Local",
      "question": "In Section 4.4, how are Knowledge Graphs used to probe knowledge stored in Large Language Models?",
      "retrieved_nodes": [
        "0000",
        "0004"
      ],
      "ground_truth_nodes": [
        "0008"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 0.25,
      "faithfulness": 0.4,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 1.2056,
      "prompt_tokens": 2224,
      "completion_tokens": 77,
      "cost_usd": 0.000597,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [4 KG-ENHANCED LLMs (Page 7)], partial evidence found (probing factual knowledge). Full evidence requires additional sections."
    },
    {
      "id": "LQ_10",
      "category": "Local",
      "question": "What specific sub-tasks are involved in Knowledge Graph Construction using LLMs in Section 5.3?",
      "retrieved_nodes": [
        "0012",
        "0009",
        "0000"
      ],
      "ground_truth_nodes": [
        "0012"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0065,
      "total_latency_s": 1.9732,
      "prompt_tokens": 2881,
      "completion_tokens": 135,
      "cost_usd": 0.000907,
      "answer": "Based on [5.3 LLM-augmented KG Construction (Page 14)], [5 LLM-AUGMENTED KGs (Page 11)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: The core sub-tasks of KG Construction are: 1) Entity Discovery (Named Entity Recognition, Entity Typing, and Entity Linking), 2) Relation Extraction (identifying relational predicates between entities), and 3) Event Extraction (identifying event triggers and event arguments)."
    },
    {
      "id": "LQ_11",
      "category": "Local",
      "question": "What is the primary challenge in Knowledge-Graph-to-Text (KG-to-text) generation discussed in Section 5.4?",
      "retrieved_nodes": [
        "0013",
        "0006"
      ],
      "ground_truth_nodes": [
        "0013"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0054,
      "total_latency_s": 1.6887,
      "prompt_tokens": 2781,
      "completion_tokens": 113,
      "cost_usd": 0.000849,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [4.2 KG-enhanced LLM Inference (Page 9)]: The primary challenge is parallel data scarcity (scarcity of aligned KG triples and natural text descriptions) and ensuring verbalization fidelity, where LLMs must generate fluent descriptions without hallucinating unsupported claims beyond the graph."
    },
    {
      "id": "LQ_12",
      "category": "Local",
      "question": "What are the two major paradigms of Knowledge Graph Question Answering (KGQA) in Section 5.5?",
      "retrieved_nodes": [
        "0014",
        "0009"
      ],
      "ground_truth_nodes": [
        "0014"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0055,
      "total_latency_s": 1.7488,
      "prompt_tokens": 2371,
      "completion_tokens": 118,
      "cost_usd": 0.000689,
      "answer": "Based on [5.5 LLM-augmented KG Question Answering (Page 17)], [5 LLM-AUGMENTED KGs (Page 11)]: The two major paradigms in KGQA are: 1) Semantic Parsing (converting natural questions into executable formal queries like SPARQL or lambda-calculus), and 2) Information Retrieval (retrieving a relevant candidate subgraph or facts and directly extracting/generating the answer)."
    },
    {
      "id": "LQ_13",
      "category": "Local",
      "question": "How does Section 2 define Commonsense Knowledge Graphs?",
      "retrieved_nodes": [
        "0000",
        "0002",
        "0003"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.25,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0061,
      "total_latency_s": 1.2927,
      "prompt_tokens": 3271,
      "completion_tokens": 84,
      "cost_usd": 0.001022,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [2 BACKGROUND (Page 2)], [3 ROADMAP & CATEGORIZATION (Page 5)], partial evidence found (everyday knowledge). Full evidence requires additional sections."
    },
    {
      "id": "LQ_14",
      "category": "Local",
      "question": "How does the General Language Model (GLM) differ from BERT and GPT in the background discussion in Section 2?",
      "retrieved_nodes": [
        "0002",
        "0000",
        "0016"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.7459,
      "prompt_tokens": 3287,
      "completion_tokens": 118,
      "cost_usd": 0.001055,
      "answer": "Based on [2 BACKGROUND (Page 2)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)]: GLM employs auto-regressive blank filling that unifies auto-encoding (BERT-style) and auto-regressive (GPT-style) objectives, allowing variable-length text span corruption and generation for multi-task applications."
    },
    {
      "id": "LQ_15",
      "category": "Local",
      "question": "What are the five key disadvantages of Large Language Models identified in Section 1 and Appendix A?",
      "retrieved_nodes": [
        "0000",
        "0020",
        "0016"
      ],
      "ground_truth_nodes": [
        "0001",
        "0020"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 2.0356,
      "prompt_tokens": 3285,
      "completion_tokens": 139,
      "cost_usd": 0.001071,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [APPENDIX A (Page 27)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)]: The five major limitations are: 1) Implicit Knowledge (uninterpretable parameter storage), 2) Hallucination (generating plausible but false statements), 3) Indecisiveness (stochastic probabilistic generation), 4) Black-box nature (lack of interpretability), and 5) Lack of Domain-specific/New Knowledge."
    },
    {
      "id": "LQ_16",
      "category": "Local",
      "question": "What are knowledge-guided training objectives in KG-enhanced pre-training (Section 4.1)?",
      "retrieved_nodes": [
        "0005",
        "0007",
        "0004"
      ],
      "ground_truth_nodes": [
        "0005"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.75,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 1.4091,
      "prompt_tokens": 2829,
      "completion_tokens": 92,
      "cost_usd": 0.000851,
      "answer": "Based on [4.1 KG-enhanced LLM Pre-training (Page 7)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4 KG-ENHANCED LLMs (Page 7)], partial evidence found (entity masking; guided by KG structure). Full evidence requires additional sections."
    },
    {
      "id": "LQ_17",
      "category": "Local",
      "question": "What are the two major sub-categories of Synergized LLMs + KGs discussed in Section 6?",
      "retrieved_nodes": [
        "0015",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0015"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 1.8592,
      "prompt_tokens": 2581,
      "completion_tokens": 126,
      "cost_usd": 0.000779,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: Section 6 categorizes Synergized LLMs + KGs into: 1) Synergized Knowledge Representation (joint pre-training and unified embeddings for text and graphs), and 2) Synergized Knowledge Reasoning (cooperative reasoning where LLMs act as cognitive agents and KGs act as symbolic verification engines)."
    },
    {
      "id": "LQ_18",
      "category": "Local",
      "question": "According to Section 7.1, how can Knowledge Graphs be utilized to detect hallucinations in LLMs?",
      "retrieved_nodes": [
        "0000",
        "0004",
        "0016"
      ],
      "ground_truth_nodes": [
        "0016"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0055,
      "total_latency_s": 1.8389,
      "prompt_tokens": 2865,
      "completion_tokens": 125,
      "cost_usd": 0.000892,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [4 KG-ENHANCED LLMs (Page 7)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)]: KGs serve as an external grounded source of truth where generated statements from LLMs are converted into structured facts and verified against reliable KG entities and paths, checking for contradictions and unsupported entity relations."
    },
    {
      "id": "LQ_19",
      "category": "Local",
      "question": "What are Multimodal Knowledge Graphs as described in Section 2?",
      "retrieved_nodes": [
        "0000",
        "0002",
        "0016"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.3333,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0063,
      "total_latency_s": 1.3996,
      "prompt_tokens": 3276,
      "completion_tokens": 92,
      "cost_usd": 0.00103,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [2 BACKGROUND (Page 2)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], partial evidence found (extends symbolic entities with multimedia). Full evidence requires additional sections."
    },
    {
      "id": "LQ_20",
      "category": "Local",
      "question": "In Section 5.2, what is the inductive relation prediction problem in Knowledge Graph Completion?",
      "retrieved_nodes": [
        "0011",
        "0009",
        "0014"
      ],
      "ground_truth_nodes": [
        "0011"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0065,
      "total_latency_s": 1.8165,
      "prompt_tokens": 3012,
      "completion_tokens": 123,
      "cost_usd": 0.000949,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [5 LLM-AUGMENTED KGs (Page 11)], [5.5 LLM-augmented KG Question Answering (Page 17)]: Inductive relation prediction involves predicting relations involving previously unseen entities that were not present during training, requiring language models to utilize textual descriptions and generalize beyond static transductive graph embeddings."
    },
    {
      "id": "GQ_01",
      "category": "Global",
      "question": "What is the overarching categorization of research unifying LLMs and KGs proposed in this roadmap?",
      "retrieved_nodes": [
        "0003",
        "0000",
        "0016"
      ],
      "ground_truth_nodes": [
        "0003",
        "0000",
        "0017"
      ],
      "recall": 0.6667,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0069,
      "total_latency_s": 2.2702,
      "prompt_tokens": 3288,
      "completion_tokens": 157,
      "cost_usd": 0.001087,
      "answer": "Based on [3 ROADMAP & CATEGORIZATION (Page 5)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)]: The paper categorizes the entire research landscape into three unified frameworks: 1) KG-enhanced LLMs (injecting KGs into LLMs during pre-training, inference, or interpretability), 2) LLM-augmented KGs (using LLMs for KGE, KGC, KG construction, KG-to-text, and KGQA), and 3) Synergized LLMs + KGs (bidirectional unification in representation and reasoning)."
    },
    {
      "id": "GQ_02",
      "category": "Global",
      "question": "What are the fundamental complementary strengths of LLMs versus KGs described throughout the paper?",
      "retrieved_nodes": [
        "0015",
        "0004",
        "0000"
      ],
      "ground_truth_nodes": [
        "0001",
        "0020",
        "0000"
      ],
      "recall": 0.3333,
      "precision": 0.3333,
      "correctness": 0.75,
      "faithfulness": 0.6,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 2.0656,
      "prompt_tokens": 2863,
      "completion_tokens": 142,
      "cost_usd": 0.000905,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: LLMs provide general knowledge, natural language understanding, generative fluency, and broad zero-shot transfer, but suffer from hallucination and black-box opacity. KGs provide structured, accurate, decisive, interpretable, and domain-specific knowledge, but lack natural language fluency and are often incomplete."
    },
    {
      "id": "GQ_03",
      "category": "Global",
      "question": "How does the paper categorize the integration of KGs into LLMs across the model lifecycle?",
      "retrieved_nodes": [
        "0004",
        "0015",
        "0009"
      ],
      "ground_truth_nodes": [
        "0008",
        "0005",
        "0003",
        "0006",
        "0004"
      ],
      "recall": 0.2,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.52,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0069,
      "total_latency_s": 1.9002,
      "prompt_tokens": 2582,
      "completion_tokens": 129,
      "cost_usd": 0.000782,
      "answer": "Based on [4 KG-ENHANCED LLMs (Page 7)], [6 SYNERGIZED LLMs + KGs (Page 18)], [5 LLM-AUGMENTED KGs (Page 11)]: The lifecycle integration covers: 1) Pre-training (enriching corpora and modifying training objectives with entity masks), 2) Inference (dynamic retrieval, prompting, and attention modifications without retraining), and 3) Interpretability (using KGs for knowledge probing and understanding model decisions)."
    },
    {
      "id": "GQ_04",
      "category": "Global",
      "question": "How do LLMs assist in addressing the incompleteness and sparseness of Knowledge Graphs across Section 5?",
      "retrieved_nodes": [
        "0000",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0003",
        "0011",
        "0009",
        "0012"
      ],
      "recall": 0.25,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.55,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 1.8722,
      "prompt_tokens": 2459,
      "completion_tokens": 127,
      "cost_usd": 0.000731,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: LLMs leverage rich textual descriptions and generalized world knowledge to enrich KG embeddings, complete missing triples via generative sequence prediction (KGC), and extract new entities, relations, and events from raw unstructured text during KG construction."
    },
    {
      "id": "GQ_05",
      "category": "Global",
      "question": "What are the future research directions and open challenges identified for unifying LLMs and KGs in Section 7?",
      "retrieved_nodes": [
        "0016",
        "0000"
      ],
      "ground_truth_nodes": [
        "0016",
        "0017"
      ],
      "recall": 0.5,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0065,
      "total_latency_s": 1.8032,
      "prompt_tokens": 2652,
      "completion_tokens": 122,
      "cost_usd": 0.000805,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: The key future directions include: 1) KGs for Hallucination Detection in LLMs, 2) Knowledge Editing using KGs, 3) Multimodal Knowledge Graph and LLM Unification, 4) Complex Multi-agent Reasoning and Synergy, and 5) Continual and Lifelong Knowledge Learning."
    },
    {
      "id": "GQ_06",
      "category": "Global",
      "question": "Why are both implicit knowledge in LLMs and explicit knowledge in KGs necessary for reliable AI according to the authors?",
      "retrieved_nodes": [
        "0000",
        "0015",
        "0004"
      ],
      "ground_truth_nodes": [
        "0001",
        "0015",
        "0003",
        "0020"
      ],
      "recall": 0.25,
      "precision": 0.3333,
      "correctness": 0.6667,
      "faithfulness": 0.55,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.008,
      "total_latency_s": 2.0213,
      "prompt_tokens": 2868,
      "completion_tokens": 138,
      "cost_usd": 0.000904,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)]: Implicit knowledge in neural parameters offers linguistic flexibility and intuitive common-sense reasoning, but lacks verifiability and precision. Explicit symbolic knowledge in KGs provides deterministic, verifiable, and editable facts. Unifying both produces reliable, grounded, and interpretable AI."
    },
    {
      "id": "GQ_07",
      "category": "Global",
      "question": "How does the taxonomy differentiate between one-way enhancement and two-way unification?",
      "retrieved_nodes": [
        "0007",
        "0015"
      ],
      "ground_truth_nodes": [
        "0003",
        "0015",
        "0004",
        "0009"
      ],
      "recall": 0.25,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 0.55,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0112,
      "total_latency_s": 1.9845,
      "prompt_tokens": 2611,
      "completion_tokens": 135,
      "cost_usd": 0.000799,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [6 SYNERGIZED LLMs + KGs (Page 18)]: One-way enhancement consists of either KG-enhanced LLMs (KG as an auxiliary feature to improve an LLM) or LLM-augmented KGs (LLM as a tool to complete or build a KG). Two-way unification (Synergized LLMs + KGs) represents a mutually reciprocal architecture where both representations and reasoning processes co-evolve."
    },
    {
      "id": "GQ_08",
      "category": "Global",
      "question": "What role do prompt engineering and in-context learning play across both KG-enhanced LLMs and LLM-augmented KGs?",
      "retrieved_nodes": [
        "0004",
        "0007",
        "0006"
      ],
      "ground_truth_nodes": [
        "0002",
        "0006",
        "0011",
        "0014"
      ],
      "recall": 0.25,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.55,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0172,
      "total_latency_s": 2.0072,
      "prompt_tokens": 2834,
      "completion_tokens": 136,
      "cost_usd": 0.000889,
      "answer": "Based on [4 KG-ENHANCED LLMs (Page 7)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.2 KG-enhanced LLM Inference (Page 9)]: In KG-enhanced LLMs, prompt engineering linearizes retrieved graph subgraphs into text prompts for in-context grounding. In LLM-augmented KGs, in-context demonstrations enable LLMs to perform link prediction, triple verbalization, and SPARQL generation without task-specific fine-tuning."
    },
    {
      "id": "GQ_09",
      "category": "Global",
      "question": "How does the paper classify Knowledge Graphs based on knowledge scope across the surveyed literature?",
      "retrieved_nodes": [
        "0000",
        "0017",
        "0002"
      ],
      "ground_truth_nodes": [
        "0002",
        "0003"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 0.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.2492,
      "prompt_tokens": 2896,
      "completion_tokens": 80,
      "cost_usd": 0.000869,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [8 CONCLUSION (Page 21)], [2 BACKGROUND (Page 2)], partial evidence found (general background). Full evidence requires additional sections."
    },
    {
      "id": "GQ_10",
      "category": "Global",
      "question": "What are the primary trade-offs between static knowledge integration (pre-training) versus dynamic knowledge integration (inference)?",
      "retrieved_nodes": [
        "0007",
        "0005"
      ],
      "ground_truth_nodes": [
        "0006",
        "0007",
        "0005"
      ],
      "recall": 0.6667,
      "precision": 1.0,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0062,
      "total_latency_s": 1.9295,
      "prompt_tokens": 2624,
      "completion_tokens": 131,
      "cost_usd": 0.000801,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.1 KG-enhanced LLM Pre-training (Page 7)]: Pre-training integrates knowledge deeply into model weights with zero retrieval latency at test time, but requires expensive retraining to update facts. Inference-stage integration is dynamic and requires no retraining, but introduces retrieval overhead, latency, and context window constraints."
    },
    {
      "id": "GQ_11",
      "category": "Global",
      "question": "How does the survey structure the role of LLMs across the full lifecycle of Knowledge Graph curation?",
      "retrieved_nodes": [
        "0004",
        "0000",
        "0015"
      ],
      "ground_truth_nodes": [
        "0009",
        "0012"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 1.0,
      "faithfulness": 0.4,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0057,
      "total_latency_s": 1.8157,
      "prompt_tokens": 2863,
      "completion_tokens": 123,
      "cost_usd": 0.00089,
      "answer": "Based on [4 KG-ENHANCED LLMs (Page 7)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [6 SYNERGIZED LLMs + KGs (Page 18)]: LLMs participate in: 1) Initial construction from raw corpora via entity, relation, and event extraction, 2) Entity linking and canonicalization, 3) Knowledge Graph Completion for missing links, and 4) Downstream KGQA and text verbalization."
    },
    {
      "id": "GQ_12",
      "category": "Global",
      "question": "What are common benchmark datasets mentioned throughout the survey for evaluating LLM and KG methods?",
      "retrieved_nodes": [
        "0007",
        "0006",
        "0012"
      ],
      "ground_truth_nodes": [
        "0014",
        "0008",
        "0005",
        "0010",
        "0011"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 0.0,
      "faithfulness": 0.4,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0076,
      "total_latency_s": 1.3876,
      "prompt_tokens": 3256,
      "completion_tokens": 91,
      "cost_usd": 0.001021,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.2 KG-enhanced LLM Inference (Page 9)], [5.3 LLM-augmented KG Construction (Page 14)], partial evidence found (general background). Full evidence requires additional sections."
    },
    {
      "id": "GQ_13",
      "category": "Global",
      "question": "How does synergized reasoning in Section 6 go beyond standard Retrieval-Augmented Generation (RAG)?",
      "retrieved_nodes": [
        "0013",
        "0015",
        "0011"
      ],
      "ground_truth_nodes": [
        "0006",
        "0015"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0057,
      "total_latency_s": 1.8657,
      "prompt_tokens": 3418,
      "completion_tokens": 127,
      "cost_usd": 0.001115,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [6 SYNERGIZED LLMs + KGs (Page 18)], [5.2 LLM-augmented KG Completion (Page 12)]: Standard RAG uses one-way linear text retrieval to supply context to an LLM. Synergized reasoning involves iterative, multi-hop symbolic path searching, where the LLM guides graph traversal and symbolic algorithms constrain and verify each step of model inference."
    },
    {
      "id": "GQ_14",
      "category": "Global",
      "question": "What are the ethical, hallucination, and factual accuracy implications discussed in the roadmap?",
      "retrieved_nodes": [
        "0000",
        "0003",
        "0016"
      ],
      "ground_truth_nodes": [
        "0016",
        "0001",
        "0004"
      ],
      "recall": 0.3333,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.6,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0134,
      "total_latency_s": 1.8601,
      "prompt_tokens": 3287,
      "completion_tokens": 126,
      "cost_usd": 0.001062,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [3 ROADMAP & CATEGORIZATION (Page 5)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)]: Hallucinations pose serious safety risks in high-stakes domains (healthcare, law). KGs offer a verifiable, traceable audit trail for model claims, enabling factual verification, reduced toxicity, and reliable hallucination detection."
    },
    {
      "id": "GQ_15",
      "category": "Global",
      "question": "How do multimodal aspects intersect with LLMs and KGs in the future vision of the paper?",
      "retrieved_nodes": [
        "0016",
        "0015",
        "0004"
      ],
      "ground_truth_nodes": [
        "0002",
        "0016"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0067,
      "total_latency_s": 1.6534,
      "prompt_tokens": 2990,
      "completion_tokens": 111,
      "cost_usd": 0.000931,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)]: The roadmap envisions unifying vision-language models with multimodal knowledge graphs (MMKGs), allowing grounding of both visual tokens and textual entities in structured multimodal knowledge for cross-modal reasoning."
    },
    {
      "id": "GQ_16",
      "category": "Global",
      "question": "How has the evolution of pre-trained models from BERT to auto-regressive models (GPT-3/4) affected KG integration strategies?",
      "retrieved_nodes": [
        "0007",
        "0013",
        "0004"
      ],
      "ground_truth_nodes": [
        "0002",
        "0001",
        "0006",
        "0005"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 1.0,
      "faithfulness": 0.4,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0066,
      "total_latency_s": 2.0099,
      "prompt_tokens": 2840,
      "completion_tokens": 137,
      "cost_usd": 0.000892,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [5.4 LLM-augmented KG-to-text Generation (Page 16)], [4 KG-ENHANCED LLMs (Page 7)]: Early models (BERT, RoBERTa) integrated KGs primarily via pre-training objective modification and embedding fusion encoders. Modern billion-parameter models shift toward inference-time prompting, in-context retrieval (RAG), and agentic tool-use, avoiding massive retraining costs."
    },
    {
      "id": "GQ_17",
      "category": "Global",
      "question": "What are the core limitations of current KG-to-text generation systems identified in Section 5.4?",
      "retrieved_nodes": [
        "0013",
        "0006"
      ],
      "ground_truth_nodes": [
        "0013"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.005,
      "total_latency_s": 1.5383,
      "prompt_tokens": 2778,
      "completion_tokens": 102,
      "cost_usd": 0.000839,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [4.2 KG-enhanced LLM Inference (Page 9)]: Core limitations include hallucinating facts not grounded in the input subgraph, difficulty linearizing complex cyclic graph topologies into sequential natural text, and handling out-of-vocabulary entities."
    },
    {
      "id": "GQ_18",
      "category": "Global",
      "question": "How does the survey summarize the transition from heuristic KG embeddings to LLM-driven embeddings in Section 5.1?",
      "retrieved_nodes": [
        "0013",
        "0010",
        "0006"
      ],
      "ground_truth_nodes": [
        "0010"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0063,
      "total_latency_s": 1.8897,
      "prompt_tokens": 3422,
      "completion_tokens": 128,
      "cost_usd": 0.001117,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [5.1 LLM-augmented KG Embedding (Page 11)], [4.2 KG-enhanced LLM Inference (Page 9)]: Early KGE methods (TransE, RotatE) relied solely on graph topology and distance heuristics, failing on sparse or unseen entities. LLM-augmented KGE incorporates rich linguistic entity descriptions, allowing semantic generalization and zero-shot entity representation."
    },
    {
      "id": "GQ_19",
      "category": "Global",
      "question": "What is the roadmap's vision for continuous knowledge updating without catastrophic forgetting?",
      "retrieved_nodes": [
        "0000",
        "0003"
      ],
      "ground_truth_nodes": [
        "0016",
        "0006",
        "0007"
      ],
      "recall": 0.0,
      "precision": 0.0,
      "correctness": 1.0,
      "faithfulness": 0.4,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0061,
      "total_latency_s": 1.7727,
      "prompt_tokens": 2646,
      "completion_tokens": 120,
      "cost_usd": 0.000801,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [3 ROADMAP & CATEGORIZATION (Page 5)]: The roadmap envisions combining dynamic retrieval-based inference with model editing techniques (Locating and Editing or ROME), using KGs to pinpoint outdated factual parameters and update them surgical-style without degrading general language capabilities."
    },
    {
      "id": "GQ_20",
      "category": "Global",
      "question": "What is the final conclusion of the survey regarding unified foundation models and structured knowledge?",
      "retrieved_nodes": [
        "0000",
        "0017",
        "0001"
      ],
      "ground_truth_nodes": [
        "0017"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0055,
      "total_latency_s": 1.7721,
      "prompt_tokens": 2897,
      "completion_tokens": 120,
      "cost_usd": 0.000901,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [8 CONCLUSION (Page 21)], [1 INTRODUCTION (Page 1)]: Section 8 concludes that LLMs and KGs are inherently complementary: neither can solve general intelligence alone. The future of AI lies in deep, synergized unification combining inductive statistical learning with deductive symbolic knowledge."
    },
    {
      "id": "MH_01",
      "category": "Multi-hop",
      "question": "How does the knowledge retrieval step in Section 4.2 relate to the KGQA information retrieval paradigm in Section 5.5?",
      "retrieved_nodes": [
        "0009",
        "0013",
        "0006"
      ],
      "ground_truth_nodes": [
        "0006",
        "0014"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.8925,
      "prompt_tokens": 3017,
      "completion_tokens": 129,
      "cost_usd": 0.000956,
      "answer": "Based on [5 LLM-AUGMENTED KGs (Page 11)], [5.4 LLM-augmented KG-to-text Generation (Page 16)], [4.2 KG-enhanced LLM Inference (Page 9)]: Both mechanisms perform entity linking on user queries to retrieve relevant subgraphs from a KG; in Section 4.2 the retrieved subgraph is converted into prompt context for text generation, while in Section 5.5 the candidate subgraph is queried to isolate the exact answer entity."
    },
    {
      "id": "MH_02",
      "category": "Multi-hop",
      "question": "Compare the computational overhead of KG-enhanced pre-training in Section 4.1 with the inference latency trade-offs analyzed in Section 4.3.",
      "retrieved_nodes": [
        "0007",
        "0005",
        "0006"
      ],
      "ground_truth_nodes": [
        "0007",
        "0005"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 2.1592,
      "prompt_tokens": 3265,
      "completion_tokens": 149,
      "cost_usd": 0.001071,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.1 KG-enhanced LLM Pre-training (Page 7)], [4.2 KG-enhanced LLM Inference (Page 9)]: Section 4.1 pre-training requires massive upfront computational resources (training on billions of tokens with multi-objective masking), but yields zero extra latency during runtime. Section 4.3 notes that inference methods have low upfront cost but add runtime latency due to entity linking and graph retrieval per query."
    },
    {
      "id": "MH_03",
      "category": "Multi-hop",
      "question": "How do the entity masking objectives in Section 4.1 compare with the entity discovery and typing methods in Section 5.3?",
      "retrieved_nodes": [
        "0012",
        "0007",
        "0010"
      ],
      "ground_truth_nodes": [
        "0005",
        "0012"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0095,
      "total_latency_s": 1.9929,
      "prompt_tokens": 3261,
      "completion_tokens": 136,
      "cost_usd": 0.001059,
      "answer": "Based on [5.3 LLM-augmented KG Construction (Page 14)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [5.1 LLM-augmented KG Embedding (Page 11)]: Entity masking in Section 4.1 uses known KG entities to mask tokens so the language model learns world facts; in contrast, Section 5.3 entity discovery applies the trained LLM in reverse to identify, type, and link new unknown entity mentions from raw text into a KG."
    },
    {
      "id": "MH_04",
      "category": "Multi-hop",
      "question": "How do the limitations of KGs in Appendix A (such as incompleteness and unseen facts) motivate the LLM-augmented KGC methods in Section 5.2?",
      "retrieved_nodes": [
        "0011",
        "0009",
        "0012"
      ],
      "ground_truth_nodes": [
        "0020",
        "0011"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0053,
      "total_latency_s": 1.8286,
      "prompt_tokens": 3021,
      "completion_tokens": 124,
      "cost_usd": 0.000954,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [5 LLM-AUGMENTED KGs (Page 11)], [5.3 LLM-augmented KG Construction (Page 14)]: Appendix A notes KGs suffer from severe incompleteness and fail on unseen facts because they ignore text. Section 5.2 directly solves this by using LLMs to read textual entity descriptions and generate missing links (PaD) or score unseen candidate entities (PaE)."
    },
    {
      "id": "MH_05",
      "category": "Multi-hop",
      "question": "How does the 'Black-box' disadvantage of LLMs in Appendix A connect to the KG-based interpretability techniques in Section 4.4?",
      "retrieved_nodes": [
        "0004",
        "0008",
        "0020"
      ],
      "ground_truth_nodes": [
        "0020",
        "0008"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0126,
      "total_latency_s": 1.7226,
      "prompt_tokens": 2997,
      "completion_tokens": 115,
      "cost_usd": 0.000937,
      "answer": "Based on [4 KG-ENHANCED LLMs (Page 7)], [4.4 KG-enhanced LLM Interpretability (Page 10)], [APPENDIX A (Page 27)]: Appendix A highlights LLMs as opaque black boxes whose reasoning cannot be verified. Section 4.4 resolves this by utilizing KGs as explicit conceptual maps to probe which facts the LLM actually knows and trace attention weights to explain decisions."
    },
    {
      "id": "MH_06",
      "category": "Multi-hop",
      "question": "In what ways do Commonsense KGs defined in Section 2 get utilized in both Pre-training (Section 4.1) and Reasoning (Section 6)?",
      "retrieved_nodes": [
        "0007",
        "0005",
        "0006"
      ],
      "ground_truth_nodes": [
        "0002",
        "0015",
        "0005"
      ],
      "recall": 0.3333,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.6,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 2.0556,
      "prompt_tokens": 3262,
      "completion_tokens": 141,
      "cost_usd": 0.001064,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.1 KG-enhanced LLM Pre-training (Page 7)], [4.2 KG-enhanced LLM Inference (Page 9)]: In Section 2, Commonsense KGs (ConceptNet, ATOMIC) capture everyday physical/social relations. In Section 4.1, they provide relational paths for entity-guided pre-training. In Section 6, they provide symbolic reasoning chains that guide LLMs through multi-hop commonsense question answering."
    },
    {
      "id": "MH_07",
      "category": "Multi-hop",
      "question": "How does Knowledge Graph Embedding in Section 5.1 provide representations that can be used for KG-enhanced inference in Section 4.2?",
      "retrieved_nodes": [
        "0006",
        "0010",
        "0007"
      ],
      "ground_truth_nodes": [
        "0010",
        "0006"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0139,
      "total_latency_s": 1.8806,
      "prompt_tokens": 3263,
      "completion_tokens": 127,
      "cost_usd": 0.001053,
      "answer": "Based on [4.2 KG-enhanced LLM Inference (Page 9)], [5.1 LLM-augmented KG Embedding (Page 11)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)]: Section 5.1 methods generate continuous entity/relation vector representations; Section 4.2 takes these low-dimensional embeddings and feeds them into the LLM's cross-attention or soft-prompt layers during inference for factual steering."
    },
    {
      "id": "MH_08",
      "category": "Multi-hop",
      "question": "How does the hallucination drawback of LLMs mentioned in Section 1 link to the future hallucination detection milestone in Section 7.1?",
      "retrieved_nodes": [
        "0016",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0016",
        "0001"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0126,
      "total_latency_s": 1.7793,
      "prompt_tokens": 2596,
      "completion_tokens": 120,
      "cost_usd": 0.000781,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: Section 1 identifies hallucination (generating factually false content) as a critical barrier to deploying LLMs. Section 7.1 establishes the research milestone of using KGs as an automated verification oracle to check LLM outputs and eliminate hallucinations."
    },
    {
      "id": "MH_09",
      "category": "Multi-hop",
      "question": "Compare how textual information is incorporated in KGC (Section 5.2) versus KG-to-text generation (Section 5.4).",
      "retrieved_nodes": [
        "0013",
        "0006",
        "0009"
      ],
      "ground_truth_nodes": [
        "0013",
        "0011"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 0.6667,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0054,
      "total_latency_s": 1.8054,
      "prompt_tokens": 3015,
      "completion_tokens": 122,
      "cost_usd": 0.00095,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [4.2 KG-enhanced LLM Inference (Page 9)], [5 LLM-AUGMENTED KGs (Page 11)]: In KGC (5.2), text descriptions are used as input features to predict missing graph links (output is a graph triple/entity). In KG-to-text (5.4), the input is a graph subgraph and the output is a coherent natural language paragraph describing the graph."
    },
    {
      "id": "MH_10",
      "category": "Multi-hop",
      "question": "How does the PaD (Pre-trained as Decoders) framework in Section 5.2 relate to prompt-based KGQA in Section 5.5?",
      "retrieved_nodes": [
        "0013",
        "0011",
        "0014"
      ],
      "ground_truth_nodes": [
        "0011",
        "0014"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0131,
      "total_latency_s": 1.9631,
      "prompt_tokens": 3425,
      "completion_tokens": 133,
      "cost_usd": 0.001123,
      "answer": "Based on [5.4 LLM-augmented KG-to-text Generation (Page 16)], [5.2 LLM-augmented KG Completion (Page 12)], [5.5 LLM-augmented KG Question Answering (Page 17)]: Both formulate the task as auto-regressive text generation via prompting: PaD prompts the LLM to generate the missing entity name for a triple (head, relation, ?), while prompt-based KGQA prompts the LLM to generate the natural answer to a user query given retrieved triples."
    },
    {
      "id": "MH_11",
      "category": "Multi-hop",
      "question": "How do structural triples in Section 2 contrast with unstructured corpora in Section 1, and how does Section 3 propose bridging them?",
      "retrieved_nodes": [
        "0011",
        "0016",
        "0003"
      ],
      "ground_truth_nodes": [
        "0002",
        "0001",
        "0003"
      ],
      "recall": 0.3333,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 0.6,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0112,
      "total_latency_s": 1.8945,
      "prompt_tokens": 3425,
      "completion_tokens": 128,
      "cost_usd": 0.001119,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [3 ROADMAP & CATEGORIZATION (Page 5)]: Section 1 describes unstructured corpora as continuous text with rich semantics but implicit facts; Section 2 describes KGs as discrete (head, relation, tail) triples. Section 3's roadmap bridges them via unified representations, dynamic retrieval, and synergized reasoning."
    },
    {
      "id": "MH_12",
      "category": "Multi-hop",
      "question": "How does domain-specific KG construction in Section 5.3 alleviate the domain-specific knowledge gap of LLMs noted in Appendix A?",
      "retrieved_nodes": [
        "0012",
        "0004",
        "0020"
      ],
      "ground_truth_nodes": [
        "0020",
        "0012"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0055,
      "total_latency_s": 1.6722,
      "prompt_tokens": 2996,
      "completion_tokens": 112,
      "cost_usd": 0.000934,
      "answer": "Based on [5.3 LLM-augmented KG Construction (Page 14)], [4 KG-ENHANCED LLMs (Page 7)], [APPENDIX A (Page 27)]: Appendix A notes general LLMs lack domain-specific expertise. Section 5.3 enables building domain-specific KGs from proprietary medical or legal texts via relation extraction, which can then be dynamically queried by LLMs during inference."
    },
    {
      "id": "MH_13",
      "category": "Multi-hop",
      "question": "Contrast the role of LLMs as feature encoders in KGE (Section 5.1) with LLMs as cognitive processors in Synergized frameworks (Section 6).",
      "retrieved_nodes": [
        "0015",
        "0011",
        "0004"
      ],
      "ground_truth_nodes": [
        "0010",
        "0015"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0085,
      "total_latency_s": 1.7552,
      "prompt_tokens": 3001,
      "completion_tokens": 118,
      "cost_usd": 0.000941,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [5.2 LLM-augmented KG Completion (Page 12)], [4 KG-ENHANCED LLMs (Page 7)]: In Section 5.1, the LLM is merely a static feature extractor that converts entity descriptions into vectors. In Section 6, the LLM is an active cognitive agent that iteratively reasons, plans graph traversals, and interacts bidirectionally with the KG."
    },
    {
      "id": "MH_14",
      "category": "Multi-hop",
      "question": "How does dynamic knowledge fusion in Section 4.2 compare with synergized bidirectional knowledge representation in Section 6?",
      "retrieved_nodes": [
        "0015",
        "0006",
        "0011"
      ],
      "ground_truth_nodes": [
        "0006",
        "0015"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0054,
      "total_latency_s": 1.8354,
      "prompt_tokens": 3421,
      "completion_tokens": 124,
      "cost_usd": 0.001114,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4.2 KG-enhanced LLM Inference (Page 9)], [5.2 LLM-augmented KG Completion (Page 12)]: Dynamic fusion in 4.2 is a one-way pipeline (retrieve KG subgraph, feed into LLM prompt or attention). Synergized representation in Section 6 trains a joint multimodal graph-text model where token embeddings and graph node embeddings inhabit a shared semantic space."
    },
    {
      "id": "MH_15",
      "category": "Multi-hop",
      "question": "How do relation extraction pipelines in Section 5.3 feed into the inductive completion methods of Section 5.2?",
      "retrieved_nodes": [
        "0011",
        "0006",
        "0009"
      ],
      "ground_truth_nodes": [
        "0011",
        "0012"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0211,
      "total_latency_s": 1.8078,
      "prompt_tokens": 3013,
      "completion_tokens": 121,
      "cost_usd": 0.000948,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [4.2 KG-enhanced LLM Inference (Page 9)], [5 LLM-AUGMENTED KGs (Page 11)]: Section 5.3 extracts new relations and entity mentions from unstructured text to build initial graph edges; Section 5.2 then takes these newly expanded subgraphs and performs inductive relation prediction to infer missing links on the newly extracted nodes."
    },
    {
      "id": "MH_16",
      "category": "Multi-hop",
      "question": "How does the evaluation of knowledge probing in Section 4.4 compare with knowledge graph completion benchmarks in Section 5.2?",
      "retrieved_nodes": [
        "0011",
        "0006",
        "0008"
      ],
      "ground_truth_nodes": [
        "0011",
        "0008"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.008,
      "total_latency_s": 1.758,
      "prompt_tokens": 3425,
      "completion_tokens": 118,
      "cost_usd": 0.001111,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [4.2 KG-enhanced LLM Inference (Page 9)], [4.4 KG-enhanced LLM Interpretability (Page 10)]: Probing (4.4) tests an LLM's frozen parametric memory on cloze prompts without graph structure at test time. KGC (5.2) evaluates graph link prediction by scoring candidate entities using both graph topology and text descriptions."
    },
    {
      "id": "MH_17",
      "category": "Multi-hop",
      "question": "How does the distinction between Encyclopedia KGs and Domain-specific KGs in Section 2 impact their integration in pre-training (Section 4.1)?",
      "retrieved_nodes": [
        "0007",
        "0005"
      ],
      "ground_truth_nodes": [
        "0002",
        "0005"
      ],
      "recall": 0.5,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0061,
      "total_latency_s": 1.8761,
      "prompt_tokens": 2626,
      "completion_tokens": 127,
      "cost_usd": 0.000798,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.1 KG-enhanced LLM Pre-training (Page 7)]: Encyclopedia KGs (Wikidata) are integrated into general-domain pre-training to boost common factual accuracy across broad benchmarks. Domain-specific KGs (UMLS) are integrated into specialized pre-trained models (e.g. BioBERT) to master technical medical/scientific vocabularies."
    },
    {
      "id": "MH_18",
      "category": "Multi-hop",
      "question": "How do the reasoning algorithms in KGs (Appendix A) complement the probabilistic generation of LLMs in Synergized Reasoning (Section 6)?",
      "retrieved_nodes": [
        "0015",
        "0020",
        "0004"
      ],
      "ground_truth_nodes": [
        "0015",
        "0020"
      ],
      "recall": 1.0,
      "precision": 0.6667,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.007,
      "total_latency_s": 1.927,
      "prompt_tokens": 2996,
      "completion_tokens": 131,
      "cost_usd": 0.000949,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [APPENDIX A (Page 27)], [4 KG-ENHANCED LLMs (Page 7)]: Appendix A notes KGs have deterministic, decisive, and interpretable symbolic reasoning algorithms, contrasting with probabilistic LLM sampling. Section 6 marries both: the LLM provides flexible natural language reasoning, while the KG's symbolic path search guarantees factual decisiveness and eliminates hallucinated steps."
    },
    {
      "id": "MH_19",
      "category": "Multi-hop",
      "question": "How do prompt engineering techniques discussed in Section 2 enable zero-shot relation extraction in Section 5.3?",
      "retrieved_nodes": [
        "0002",
        "0011",
        "0014"
      ],
      "ground_truth_nodes": [
        "0002",
        "0012"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0066,
      "total_latency_s": 2.06,
      "prompt_tokens": 3418,
      "completion_tokens": 141,
      "cost_usd": 0.001126,
      "answer": "Based on [2 BACKGROUND (Page 2)], [5.2 LLM-augmented KG Completion (Page 12)], [5.5 LLM-augmented KG Question Answering (Page 17)]: Section 2 explains that prompt engineering formats downstream tasks into natural language prompts; Section 5.3 utilizes this by formulating relation extraction as a question-answering or masked completion prompt (e.g., 'What is the relationship between X and Y?'), enabling zero-shot relation extraction without labeled training data."
    },
    {
      "id": "MH_20",
      "category": "Multi-hop",
      "question": "Why does knowledge obsolescence discussed in Section 4.3 motivate the future direction of Knowledge Editing in Section 7?",
      "retrieved_nodes": [
        "0016",
        "0004",
        "0006"
      ],
      "ground_truth_nodes": [
        "0016",
        "0007"
      ],
      "recall": 0.5,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.6667,
      "retrieval_latency_s": 0.0067,
      "total_latency_s": 1.8167,
      "prompt_tokens": 2999,
      "completion_tokens": 123,
      "cost_usd": 0.000944,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [4 KG-ENHANCED LLMs (Page 7)], [4.2 KG-enhanced LLM Inference (Page 9)]: Section 4.3 highlights that real-world knowledge changes constantly, making static pre-trained models outdated. Section 7 identifies Knowledge Editing as the required future breakthrough to locate and alter specific factual weights in an LLM when KG facts update."
    },
    {
      "id": "SQ_01",
      "category": "Summary",
      "question": "Provide a comprehensive summary of the 'KG-enhanced LLMs' framework presented in Section 4.",
      "retrieved_nodes": [
        "0004",
        "0005",
        "0008"
      ],
      "ground_truth_nodes": [
        "0008",
        "0005",
        "0006",
        "0007",
        "0004"
      ],
      "recall": 0.6,
      "precision": 1.0,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0102,
      "total_latency_s": 2.4069,
      "prompt_tokens": 2993,
      "completion_tokens": 167,
      "cost_usd": 0.000977,
      "answer": "Based on [4 KG-ENHANCED LLMs (Page 7)], [4.1 KG-enhanced LLM Pre-training (Page 7)], [4.4 KG-enhanced LLM Interpretability (Page 10)]: KG-enhanced LLMs integrate structured knowledge into language models across three stages: 1) Pre-training (incorporating entity embeddings and knowledge-guided objectives like entity masking), 2) Inference (dynamically retrieving relevant subgraphs and injecting them via prompt templates or visibility attention without retraining), and 3) Interpretability (using KGs to probe internal factual knowledge and trace decision pathways)."
    },
    {
      "id": "SQ_02",
      "category": "Summary",
      "question": "Provide a comprehensive summary of the 'LLM-augmented KGs' framework presented in Section 5.",
      "retrieved_nodes": [
        "0009",
        "0012",
        "0011"
      ],
      "ground_truth_nodes": [
        "0013",
        "0014",
        "0009",
        "0010",
        "0012",
        "0011"
      ],
      "recall": 0.5,
      "precision": 1.0,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 2.4958,
      "prompt_tokens": 3009,
      "completion_tokens": 174,
      "cost_usd": 0.000989,
      "answer": "Based on [5 LLM-AUGMENTED KGs (Page 11)], [5.3 LLM-augmented KG Construction (Page 14)], [5.2 LLM-augmented KG Completion (Page 12)]: LLM-augmented KGs leverages LLMs to enhance structural knowledge graphs across five core areas: 1) KG Embedding (using LLMs as text encoders for entities), 2) KG Completion (inferring missing links via PaE or PaD generative decoding), 3) KG Construction (discovering entities, relations, and events from raw text), 4) KG-to-text generation (verbalizing subgraphs into fluent natural language), and 5) KGQA (semantic parsing and information retrieval over graphs)."
    },
    {
      "id": "SQ_03",
      "category": "Summary",
      "question": "Summarize the 'Synergized LLMs + KGs' framework and its two main paradigms in Section 6.",
      "retrieved_nodes": [
        "0015",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0015"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0079,
      "total_latency_s": 2.1912,
      "prompt_tokens": 2582,
      "completion_tokens": 151,
      "cost_usd": 0.0008,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: Synergized LLMs + KGs represents bidirectional unification with two main paradigms: 1) Synergized Knowledge Representation, where models jointly learn unified representations of text and graphs in a shared semantic space, and 2) Synergized Knowledge Reasoning, where LLMs act as intuitive cognitive reasoning engines and KGs act as symbolic, verifiable knowledge anchors for multi-hop deduction."
    },
    {
      "id": "SQ_04",
      "category": "Summary",
      "question": "Summarize the Roadmap and categorization scheme introduced in Section 3.",
      "retrieved_nodes": [
        "0003",
        "0000"
      ],
      "ground_truth_nodes": [
        "0003"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 0.6667,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0051,
      "total_latency_s": 2.0451,
      "prompt_tokens": 2641,
      "completion_tokens": 140,
      "cost_usd": 0.000815,
      "answer": "Based on [3 ROADMAP & CATEGORIZATION (Page 5)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: Section 3 organizes the entire landscape into a systematic roadmap: 1) KG-enhanced LLMs (pre-training, inference, interpretability), 2) LLM-augmented KGs (embedding, completion, construction, KG-to-text, KGQA), and 3) Synergized LLMs + KGs (representation and reasoning), mapping how models evolve from one-way enhancement to full synergy."
    },
    {
      "id": "SQ_05",
      "category": "Summary",
      "question": "Summarize the comparison between KG-enhanced Pre-training and Inference in Section 4.3.",
      "retrieved_nodes": [
        "0007",
        "0006",
        "0005"
      ],
      "ground_truth_nodes": [
        "0007"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0064,
      "total_latency_s": 2.1997,
      "prompt_tokens": 3252,
      "completion_tokens": 152,
      "cost_usd": 0.001069,
      "answer": "Based on [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.2 KG-enhanced LLM Inference (Page 9)], [4.1 KG-enhanced LLM Pre-training (Page 7)]: Pre-training aligns knowledge deeply into model weights with zero test-time retrieval overhead, but is computationally expensive and cannot be easily updated. Inference methods require no retraining and support real-time dynamic knowledge updates, but introduce inference latency, retrieval complexity, and context window bottlenecks."
    },
    {
      "id": "SQ_06",
      "category": "Summary",
      "question": "Summarize the sub-tasks and LLM roles in Knowledge Graph Construction (Section 5.3).",
      "retrieved_nodes": [
        "0012",
        "0009",
        "0000"
      ],
      "ground_truth_nodes": [
        "0012"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0055,
      "total_latency_s": 2.1655,
      "prompt_tokens": 2879,
      "completion_tokens": 149,
      "cost_usd": 0.000917,
      "answer": "Based on [5.3 LLM-augmented KG Construction (Page 14)], [5 LLM-AUGMENTED KGs (Page 11)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: KG Construction covers three sub-tasks: 1) Entity Discovery (NER, Entity Typing, and Entity Linking), 2) Relation Extraction (identifying predicates between entities), and 3) Event Extraction (identifying event triggers and arguments). LLMs excel in all three via prompt-based zero-shot and few-shot extraction from unstructured text."
    },
    {
      "id": "SQ_07",
      "category": "Summary",
      "question": "Summarize the pros and cons of LLMs and KGs as detailed in Appendix A.",
      "retrieved_nodes": [
        "0020",
        "0015",
        "0000"
      ],
      "ground_truth_nodes": [
        "0020"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 2.169,
      "prompt_tokens": 3274,
      "completion_tokens": 149,
      "cost_usd": 0.001075,
      "answer": "Based on [APPENDIX A (Page 27)], [6 SYNERGIZED LLMs + KGs (Page 18)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: LLM pros: General Knowledge, Language Processing, Generalizability. LLM cons: Implicit Knowledge, Hallucination, Indecisiveness, Black-box nature, Lacking Domain Knowledge. KG pros: Structural Knowledge, Accuracy, Decisiveness, Interpretability, Domain Knowledge, Evolving Knowledge. KG cons: Incompleteness, Lacking Language Understanding, Unseen Facts."
    },
    {
      "id": "SQ_08",
      "category": "Summary",
      "question": "Summarize the key future directions and milestones outlined in Section 7.",
      "retrieved_nodes": [
        "0016",
        "0009"
      ],
      "ground_truth_nodes": [
        "0016"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0063,
      "total_latency_s": 1.8397,
      "prompt_tokens": 2365,
      "completion_tokens": 125,
      "cost_usd": 0.000692,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [5 LLM-AUGMENTED KGs (Page 11)]: Section 7 highlights key frontiers: 1) Using KGs as ground truth to detect and eliminate LLM hallucinations, 2) Knowledge Editing to surgically update parameters, 3) Multimodal graph and vision-language unification, 4) Complex multi-hop reasoning agents, and 5) Lifelong continually updated unified AI systems."
    },
    {
      "id": "SQ_09",
      "category": "Summary",
      "question": "Summarize the background and classification of Knowledge Graphs in Section 2.",
      "retrieved_nodes": [
        "0002",
        "0000",
        "0016"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.4,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0056,
      "total_latency_s": 1.4289,
      "prompt_tokens": 3279,
      "completion_tokens": 94,
      "cost_usd": 0.001033,
      "answer": "Based on [2 BACKGROUND (Page 2)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], partial evidence found (triples: head, relation, tail; Domain-specific KGs). Full evidence requires additional sections."
    },
    {
      "id": "SQ_10",
      "category": "Summary",
      "question": "Summarize the main conclusions of the entire paper presented in Section 8.",
      "retrieved_nodes": [
        "0009",
        "0017",
        "0004"
      ],
      "ground_truth_nodes": [
        "0017"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0057,
      "total_latency_s": 1.759,
      "prompt_tokens": 2192,
      "completion_tokens": 119,
      "cost_usd": 0.000618,
      "answer": "Based on [5 LLM-AUGMENTED KGs (Page 11)], [8 CONCLUSION (Page 21)], [4 KG-ENHANCED LLMs (Page 7)]: Section 8 concludes that unifying LLMs and KGs bridges statistical connectionist AI and symbolic deterministic AI. The survey has provided a full roadmap, demonstrated how both paradigms address each other's weaknesses, and provided a structured foundation for future research."
    },
    {
      "id": "NF_01",
      "category": "Numerical/Factual",
      "question": "Which specific research grants funded this paper according to the Acknowledgments section?",
      "retrieved_nodes": [
        "0018",
        "0006",
        "0004"
      ],
      "ground_truth_nodes": [
        "0018"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.5259,
      "prompt_tokens": 2415,
      "completion_tokens": 101,
      "cost_usd": 0.000693,
      "answer": "Based on [ACKNOWLEDGMENTS (Page 21)], [4.2 KG-enhanced LLM Inference (Page 9)], [4 KG-ENHANCED LLMs (Page 7)]: This research was supported by the Australian Research Council (ARC) under grants FT210100097 and DP240101547, and the National Natural Science Foundation of China (NSFC) under grant 62120106008."
    },
    {
      "id": "NF_02",
      "category": "Numerical/Factual",
      "question": "In Section 2, what are the four representative Encyclopedia Knowledge Graphs explicitly named in the text?",
      "retrieved_nodes": [
        "0000",
        "0002",
        "0009"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.2725,
      "prompt_tokens": 2879,
      "completion_tokens": 82,
      "cost_usd": 0.000863,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [2 BACKGROUND (Page 2)], [5 LLM-AUGMENTED KGs (Page 11)], partial evidence found (general background). Full evidence requires additional sections."
    },
    {
      "id": "NF_03",
      "category": "Numerical/Factual",
      "question": "According to Section 2, what are the two representative Commonsense Knowledge Graphs explicitly mentioned?",
      "retrieved_nodes": [
        "0000",
        "0002",
        "0009"
      ],
      "ground_truth_nodes": [
        "0002"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 0.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.0,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 1.2725,
      "prompt_tokens": 2879,
      "completion_tokens": 82,
      "cost_usd": 0.000863,
      "answer": "Based on [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [2 BACKGROUND (Page 2)], [5 LLM-AUGMENTED KGs (Page 11)], partial evidence found (general background). Full evidence requires additional sections."
    },
    {
      "id": "NF_04",
      "category": "Numerical/Factual",
      "question": "What is the exact paper title and who are the authors listed in Node 0000?",
      "retrieved_nodes": [
        "0016",
        "0007",
        "0000"
      ],
      "ground_truth_nodes": [
        "0000"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0062,
      "total_latency_s": 1.8295,
      "prompt_tokens": 3121,
      "completion_tokens": 124,
      "cost_usd": 0.000994,
      "answer": "Based on [7 FUTURE DIRECTIONS AND MILESTONES (Page 20)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: Title: 'Unifying Large Language Models and Knowledge Graphs: A Roadmap'. Authors: Shirui Pan, Senior Member, IEEE; Linhao Luo; Yufei Wang; Chen Chen; Jiapu Wang; Xindong Wu, Fellow, IEEE."
    },
    {
      "id": "NF_05",
      "category": "Numerical/Factual",
      "question": "In Section 5.1, what are the two main categories of LLM-augmented KGE methods?",
      "retrieved_nodes": [
        "0011",
        "0010",
        "0009"
      ],
      "ground_truth_nodes": [
        "0010"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 1.0,
      "retrieval_latency_s": 0.0062,
      "total_latency_s": 1.3128,
      "prompt_tokens": 3005,
      "completion_tokens": 85,
      "cost_usd": 0.000916,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [5.1 LLM-augmented KG Embedding (Page 11)], [5 LLM-AUGMENTED KGs (Page 11)]: The two main categories are: 1) LLM as Entity/Relation Encoders, and 2) Joint Structural and Textual Embedding."
    },
    {
      "id": "NF_06",
      "category": "Numerical/Factual",
      "question": "In Section 5.2, what do the abbreviations PaE and PaD stand for in Knowledge Graph Completion?",
      "retrieved_nodes": [
        "0011",
        "0000",
        "0009"
      ],
      "ground_truth_nodes": [
        "0011"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 1.3758,
      "prompt_tokens": 2881,
      "completion_tokens": 90,
      "cost_usd": 0.000871,
      "answer": "Based on [5.2 LLM-augmented KG Completion (Page 12)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)], [5 LLM-AUGMENTED KGs (Page 11)], partial evidence found (Pre-trained as Encoders (PaE)). Full evidence requires additional sections."
    },
    {
      "id": "NF_07",
      "category": "Numerical/Factual",
      "question": "In Section 4.1, what are the three distinct categories of KG-enhanced pre-training approaches?",
      "retrieved_nodes": [
        "0005",
        "0007",
        "0006"
      ],
      "ground_truth_nodes": [
        "0005"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0058,
      "total_latency_s": 1.6258,
      "prompt_tokens": 3253,
      "completion_tokens": 109,
      "cost_usd": 0.001035,
      "answer": "Based on [4.1 KG-enhanced LLM Pre-training (Page 7)], [4.3 Comparison between KG-enhanced LLM Pre-training and Inference (Page 9)], [4.2 KG-enhanced LLM Inference (Page 9)]: The three categories are: 1) KG-enhanced Training Corpus, 2) Knowledge-guided Training Objectives, and 3) KG-enhanced Architecture (or Knowledge-fusion Encoders)."
    },
    {
      "id": "NF_08",
      "category": "Numerical/Factual",
      "question": "According to Section 3.1 and Fig. 6, how many primary unified frameworks are identified in the roadmap?",
      "retrieved_nodes": [
        "0003",
        "0000"
      ],
      "ground_truth_nodes": [
        "0003"
      ],
      "recall": 1.0,
      "precision": 0.5,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.5,
      "retrieval_latency_s": 0.0065,
      "total_latency_s": 1.3198,
      "prompt_tokens": 2648,
      "completion_tokens": 86,
      "cost_usd": 0.000774,
      "answer": "Based on [3 ROADMAP & CATEGORIZATION (Page 5)], [Unifying Large Language Models and Knowledge Graphs: A Roadmap (Page 1)]: Three primary unified frameworks are identified: 1) KG-enhanced LLMs, 2) LLM-augmented KGs, and 3) Synergized LLMs + KGs."
    },
    {
      "id": "NF_09",
      "category": "Numerical/Factual",
      "question": "In Section 5.3, what are the four specific subtasks identified under Entity Discovery for Knowledge Graph Construction?",
      "retrieved_nodes": [
        "0012",
        "0003",
        "0009"
      ],
      "ground_truth_nodes": [
        "0012"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0059,
      "total_latency_s": 1.4059,
      "prompt_tokens": 3015,
      "completion_tokens": 92,
      "cost_usd": 0.000926,
      "answer": "Based on [5.3 LLM-augmented KG Construction (Page 14)], [3 ROADMAP & CATEGORIZATION (Page 5)], [5 LLM-AUGMENTED KGs (Page 11)]: Entity Discovery comprises: 1) Named Entity Recognition (NER), 2) Entity Typing, and 3) Entity Linking (or Entity Alignment / Disambiguation)."
    },
    {
      "id": "NF_10",
      "category": "Numerical/Factual",
      "question": "In Section 6, what are the two main branches of Synergized LLMs + KGs?",
      "retrieved_nodes": [
        "0015",
        "0004",
        "0009"
      ],
      "ground_truth_nodes": [
        "0015"
      ],
      "recall": 1.0,
      "precision": 0.3333,
      "correctness": 1.0,
      "faithfulness": 1.0,
      "citation_accuracy": 0.3333,
      "retrieval_latency_s": 0.0053,
      "total_latency_s": 1.222,
      "prompt_tokens": 2577,
      "completion_tokens": 78,
      "cost_usd": 0.000739,
      "answer": "Based on [6 SYNERGIZED LLMs + KGs (Page 18)], [4 KG-ENHANCED LLMs (Page 7)], [5 LLM-AUGMENTED KGs (Page 11)]: The two main branches are: 1) Synergized Knowledge Representation, and 2) Synergized Knowledge Reasoning."
    }
  ]
}