Download index.html from thinkwee/DDR_Bench: direct link, hf CLI and curl.
- Browser
- Download file 40.9 kB
-
https://huggingface.co/spaces/thinkwee/DDR_Bench/resolve/main/index.html
- Command line
-
hf download hf://spaces/thinkwee/DDR_Bench/index.html
-
curl -L -o index.html https://huggingface.co/spaces/thinkwee/DDR_Bench/resolve/main/index.html
40.9 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <meta name="description" content="DDR-Bench: A Deep Data Research Agent Benchmark for LLMs"> | |
| <title>DDR-Bench | Deep Data Research Benchmark</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script> | |
| <script src="https://cdn.jsdelivr.net/npm/marked@4.3.0/marked.min.js"></script> | |
| <link rel="stylesheet" | |
| href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css"> | |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script> | |
| <script src="https://unpkg.com/sql-formatter@4.0.2/dist/sql-formatter.min.js"></script> | |
| <script src="data.js" defer></script> | |
| <script src="entropy_data.js" defer></script> | |
| <script src="trajectory_data.js" defer></script> | |
| <script src="trajectory.js" defer></script> | |
| <script src="benchmarking_data.js" defer></script> | |
| <script src="benchmarking.js" defer></script> | |
| <script src="charts.js" defer></script> | |
| <link rel="stylesheet" href="styles.css?v=3"> | |
| <style> | |
| /* Inline critical CSS for chart loading states */ | |
| .chart-loading { | |
| display: flex; | |
| align-items: center; | |
| justify-content: center; | |
| min-height: 300px; | |
| color: var(--color-text-muted, #64748B); | |
| font-size: 14px; | |
| } | |
| .chart-loading::after { | |
| content: 'Loading chart...'; | |
| animation: pulse 1.5s ease-in-out infinite; | |
| } | |
| @keyframes pulse { | |
| 0%, | |
| 100% { | |
| opacity: 0.4; | |
| } | |
| 50% { | |
| opacity: 1; | |
| } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <header class="hero"> | |
| <div class="hero-content"> | |
| <!-- <h1><img src="assets/logo.png" alt="DDR-Bench Logo" class="title-logo">DDR-Bench</h1> --> | |
| <img src="assets/social_preview.png" alt="DDR-Bench - Deep Data Research" class="hero-preview-img"> | |
| <h2>Hunt Instead of Wait: Evaluating Deep Data Research on Large Language Models</h2> | |
| <p class="description"> | |
| We distinguish <em>investigatory intelligence</em> (autonomously setting goals and exploring) from | |
| <em>executional intelligence</em> (completing assigned tasks), arguing that true agency requires the | |
| former. | |
| To evaluate this, we introduce <strong>Deep Data Research (DDR)</strong>, an open-ended task where LLMs | |
| autonomously extract insights from databases, and <strong>DDR-Bench</strong>, a large-scale, | |
| checklist-based benchmark enabling verifiable evaluation. | |
| Results show that while frontier models display emerging agency, long-horizon exploration remains | |
| challenging, with effective investigatory intelligence depending on intrinsic agentic strategies beyond | |
| mere scaffolding or scaling. | |
| </p> | |
| <div class="meta-info"> | |
| <div class="meta-row authors"> | |
| <span class="meta-item"> | |
| <a href="https://thinkwee.top/about" target="_blank" rel="noopener noreferrer">Wei Liu</a>, | |
| <a href="https://github.com/yupeijei1997" target="_blank" rel="noopener noreferrer">Peijie | |
| Yu</a>, | |
| <a href="https://www.kcl.ac.uk/people/michele-orini" target="_blank" | |
| rel="noopener noreferrer">Michele Orini</a>, | |
| <a href="https://yalidu.github.io/" target="_blank" rel="noopener noreferrer">Yali Du</a>, | |
| <a href="https://sites.google.com/view/yulanhe/home" target="_blank" | |
| rel="noopener noreferrer">Yulan He</a> | |
| </span> | |
| </div> | |
| <div class="meta-row affiliations"> | |
| <a href="https://kclnlp.github.io/" target="_blank" rel="noopener noreferrer"> | |
| <img src="assets/kcl.svg" alt="King's College London" class="affiliation-logo kcl-logo"> | |
| </a> | |
| <a href="https://www.tencent.com/en-us/" target="_blank" rel="noopener noreferrer"> | |
| <img src="assets/tencent.png" alt="Tencent" class="affiliation-logo"> | |
| </a> | |
| <a href="https://www.turing.ac.uk/" target="_blank" rel="noopener noreferrer"> | |
| <img src="assets/alan.png" alt="The Alan Turing Institute" class="affiliation-logo"> | |
| </a> | |
| </div> | |
| <div class="meta-row links"> | |
| <a href="https://huggingface.co/collections/thinkwee/ddrbench" class="platform-btn dataset-btn" | |
| target="_blank" rel="noopener noreferrer"> | |
| <svg viewBox="0 0 24 24" width="30" height="30" fill="none" stroke="currentColor" | |
| stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <ellipse cx="12" cy="5" rx="9" ry="3" /> | |
| <path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5" /> | |
| <path d="M3 12c0 1.66 4 3 9 3s9-1.34 9-3" /> | |
| </svg> | |
| Dataset | |
| </a> | |
| <a href="https://github.com/thinkwee/DDR_Bench" class="platform-btn github-btn" target="_blank" | |
| rel="noopener noreferrer"> | |
| <svg viewBox="0 0 24 24" width="30" height="30" fill="currentColor"> | |
| <path | |
| d="M12 2C6.477 2 2 6.477 2 12c0 4.42 2.865 8.17 6.839 9.49.5.092.682-.217.682-.482 0-.237-.008-.866-.013-1.7-2.782.603-3.369-1.34-3.369-1.34-.454-1.156-1.11-1.463-1.11-1.463-.908-.62.069-.608.069-.608 1.003.07 1.531 1.03 1.531 1.03.892 1.529 2.341 1.087 2.91.831.092-.646.35-1.086.636-1.336-2.22-.253-4.555-1.11-4.555-4.943 0-1.091.39-1.984 1.029-2.683-.103-.253-.446-1.27.098-2.647 0 0 .84-.269 2.75 1.025A9.578 9.578 0 0112 6.836c.85.004 1.705.114 2.504.336 1.909-1.294 2.747-1.025 2.747-1.025.546 1.377.203 2.394.1 2.647.64.699 1.028 1.592 1.028 2.683 0 3.842-2.339 4.687-4.566 4.935.359.309.678.919.678 1.852 0 1.336-.012 2.415-.012 2.743 0 .267.18.578.688.48C19.138 20.167 22 16.418 22 12c0-5.523-4.477-10-10-10z" /> | |
| </svg> | |
| Code | |
| </a> | |
| <a href="https://huggingface.co/papers/2602.02039" class="platform-btn huggingface-btn" | |
| target="_blank" rel="noopener noreferrer"> | |
| <img src="assets/hf-logo-pirate.svg" alt="HuggingFace" width="30" height="30" | |
| class="platform-icon"> | |
| HuggingFace | |
| </a> | |
| <a href="https://arxiv.org/abs/2602.02039" class="platform-btn arxiv-btn" target="_blank" | |
| rel="noopener noreferrer"> | |
| <img src="assets/arxiv-logomark-small.svg" alt="arXiv" width="30" height="30" | |
| class="platform-icon"> | |
| arXiv | |
| </a> | |
| <a href="https://www.alphaxiv.org/abs/2602.02039" class="platform-btn alphaxiv-btn" target="_blank" | |
| rel="noopener noreferrer"> | |
| <img src="assets/alphaxiv_logo.png" alt="AlphaXiv" width="30" height="30" class="platform-icon"> | |
| AlphaXiv | |
| </a> | |
| <a href="https://thinkwee.notion.site/ddrbench" class="platform-btn notion-btn" target="_blank" | |
| rel="noopener noreferrer"> | |
| <svg viewBox="0 0 24 24" width="30" height="30" fill="currentColor"> | |
| <path | |
| d="M19 3H5c-1.103 0-2 .897-2 2v14c0 1.103.897 2 2 2h14c1.103 0 2-.897 2-2V5c0-1.103-.897-2-2-2zM9 17H7.17V7H9l5.83 6.91V7H16.83v10H15L9.17 10.09V17z" /> | |
| </svg> | |
| Notion Blog | |
| </a> | |
| </div> | |
| </div> | |
| </div> | |
| </header> | |
| <!-- Main Content - All sections visible --> | |
| <main class="content"> | |
| <!-- 1. Framework Overview Section --> | |
| <section id="framework" class="section visible framework-section"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <rect width="18" height="18" x="3" y="3" rx="2" ry="2" /> | |
| <line x1="3" x2="21" y1="9" y2="9" /> | |
| <line x1="9" x2="9" y1="21" y2="9" /> | |
| </svg> | |
| Framework Overview | |
| </h2> | |
| <p>Overview of DDR-Bench.</p> | |
| </div> | |
| <div class="framework-grid"> | |
| <div class="framework-card"> | |
| <div class="framework-img-wrapper"> | |
| <div class="skeleton-loader"></div> | |
| <img src="assets/framework_task.png" alt="Task Formulation Framework" class="framework-img" | |
| loading="lazy" | |
| onload="this.classList.add('loaded'); this.previousElementSibling.style.display='none';"> | |
| </div> | |
| <h3>Task Formulation</h3> | |
| <p class="framework-description">A case of Claude Sonnet 4.5's trajectory and evaluation checklist | |
| in the MIMIC scenario of DDR-Bench. Verified fact and supporting insights are | |
| <u>underlined</u>. The agent is asked to perform multiple ReAct turns to explore the database | |
| without predefined targets or queries, autonomously mine insights from the exploration. | |
| </p> | |
| </div> | |
| <div class="framework-card"> | |
| <div class="framework-img-wrapper"> | |
| <div class="skeleton-loader"></div> | |
| <img src="assets/framework_pipeline.png" alt="Evaluation Pipeline Framework" | |
| class="framework-img" loading="lazy" | |
| onload="this.classList.add('loaded'); this.previousElementSibling.style.display='none';"> | |
| </div> | |
| <h3>Evaluation Pipeline</h3> | |
| <p class="framework-description"><b>Left</b>: Compared with previous tasks, <i>DDR</i> maximises | |
| exploration openness and agency, focusing on the direct evaluation of insight quality. | |
| <b>Right</b>: Overview of the DDR-Bench. The checklist derived from the freeform parts of the | |
| database is used to evaluate the agent generated insights from the exploration on the structured | |
| parts of the database. | |
| </p> | |
| </div> | |
| </div> | |
| </section> | |
| <!-- 1.5. Agent Trajectory Section --> | |
| <section id="trajectory" class="section visible trajectory-section"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <polyline points="22 12 18 12 15 21 9 3 6 12 2 12"></polyline> | |
| </svg> | |
| Agent Trajectory | |
| </h2> | |
| <p>Observe the autonomous decision-making process of the agent across different scenarios.</p> | |
| </div> | |
| <div class="dimension-toggle"> | |
| <button class="dim-btn active" data-traj-scenario="mimic">MIMIC</button> | |
| <button class="dim-btn" data-traj-scenario="10k">10-K</button> | |
| <button class="dim-btn" data-traj-scenario="globem">GLOBEM</button> | |
| </div> | |
| <p id="trajectory-scenario-description" class="trajectory-description"> | |
| Exploring clinical patterns and patient outcomes in a large-scale electronic health record (EHR) | |
| database. | |
| </p> | |
| <div class="trajectory-container"> | |
| <div id="chat-window" class="chat-window"> | |
| <!-- Messages will be injected here via JS --> | |
| <div class="loading-message">Loading trajectory data...</div> | |
| </div> | |
| <div class="scroll-hint" id="scroll-hint"> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M12 5v14M19 12l-7 7-7-7" /> | |
| </svg> | |
| <span>Scroll to see more</span> | |
| </div> | |
| </div> | |
| </section> | |
| <!-- 1.75. Benchmarking Section --> | |
| <section id="benchmarking" class="section visible benchmarking-section"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <line x1="12" x2="12" y1="20" y2="10" /> | |
| <line x1="18" x2="18" y1="20" y2="4" /> | |
| <line x1="6" x2="6" y1="20" y2="16" /> | |
| </svg> | |
| Benchmarking | |
| </h2> | |
| <p>Overall average accuracy across all scenarios and evaluation metrics. | |
| <br> | |
| <span class="model-badge proprietary">Purple = Proprietary</span> | |
| <span class="model-badge opensource">Green = Open-source</span> | |
| </p> | |
| </div> | |
| <div class="charts-grid single"> | |
| <div class="chart-card wide"> | |
| <div id="benchmarking-chart" class="chart-container-benchmarking"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">Claude 4.5 Sonnet achieves the highest overall average accuracy at 47.73%, | |
| significantly outperforming other models. Among open-source models, DeepSeek-V3.2 leads with 38.80%, | |
| followed closely by GLM-4.6 (37.52%) and Kimi K2 (36.42%). The results demonstrate a clear performance | |
| gap between frontier proprietary models and open-source alternatives, though top open-source models | |
| remain competitive with mid-tier proprietary offerings.</p> | |
| </section> | |
| <!-- 2. Experiment Results Section --> | |
| <section id="results" class="section visible results-section"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M3 3v18h18" /> | |
| <path d="m19 9-5 5-4-4-3 3" /> | |
| </svg> | |
| Experiments | |
| </h2> | |
| <p>Main benchmark results and in-depth analysis of agent capabilities.</p> | |
| </div> | |
| <!-- Carousel Container --> | |
| <div class="carousel-wrapper"> | |
| <button class="carousel-btn carousel-prev" aria-label="Previous"> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="m15 18-6-6 6-6" /> | |
| </svg> | |
| </button> | |
| <div class="carousel-track" id="results-carousel"> | |
| <!-- 1. Overall --> | |
| <div class="carousel-card"> | |
| <img src="assets/overall.png" alt="Overall Performance"> | |
| <h4>Overall Performance</h4> | |
| <p class="card-caption">Systematic evaluation of mainstream LLMs across MIMIC, 10-K, and GLOBEM | |
| datasets reveals persistent limitations in frontier models.</p> | |
| </div> | |
| <!-- 2. Qwen Family --> | |
| <div class="carousel-card"> | |
| <img src="assets/qwenfamily.png" alt="Qwen Family Performance"> | |
| <h4>Training-time Factors Analysis</h4> | |
| <p class="card-caption">Training-time factors study within the Qwen family. From left to right, | |
| the three columns examine inference-time scaling performance across all scenarios for models | |
| with different parameter scales, context optimisation methods, and model generations with | |
| different training strategies.</p> | |
| </div> | |
| <!-- 3. Reasoning --> | |
| <div class="carousel-card"> | |
| <img src="assets/reasoning.png" alt="Reasoning Budget"> | |
| <h4>Reasoning Budget</h4> | |
| <p class="card-caption">Increasing the reasoning budget reduces interaction rounds but | |
| illustrates | |
| a | |
| trade-off between reasoning depth and exploration efficiency.</p> | |
| </div> | |
| <!-- 4. Memory --> | |
| <div class="carousel-card"> | |
| <img src="assets/memory.png" alt="Memory Mechanism"> | |
| <h4>Memory Mechanism</h4> | |
| <p class="card-caption">Long-short-term memory can create unpredictable behavior, often | |
| increasing | |
| tool usage without consistently improving final accuracy.</p> | |
| </div> | |
| <!-- 5. Agency --> | |
| <div class="carousel-card"> | |
| <img src="assets/agency.png" alt="Proactive vs Reactive"> | |
| <h4>Proactive vs Reactive</h4> | |
| <p class="card-caption">Models perform significantly better with explicit queries (Reactive), | |
| highlighting the difficulty of true proactive goal formulation.</p> | |
| </div> | |
| <!-- 6. Hallucination --> | |
| <div class="carousel-card"> | |
| <img src="assets/hallucination.png" alt="Hallucination Analysis"> | |
| <h4>Hallucination Analysis</h4> | |
| <p class="card-caption">Hallucination rates (%) across models in DDR-Bench, measured as the | |
| proportion of insights containing factual but unfaithful information that are not derivable | |
| from the provided inputs, which is low.</p> | |
| </div> | |
| <!-- 6.5 Hallucination-Accuracy Correlation --> | |
| <div class="carousel-card"> | |
| <img src="assets/hallu_acc_corr.png" alt="Hallucination-Accuracy Correlation"> | |
| <h4>Hallucination-Accuracy Correlation</h4> | |
| <p class="card-caption">Hallucination rates show almost no correlation with final accuracy, | |
| indicating | |
| robustness against metric inflation via memorization.</p> | |
| </div> | |
| <!-- 7. Trustworthiness --> | |
| <div class="carousel-card"> | |
| <img src="assets/trustworthiness.png" alt="Trustworthiness"> | |
| <h4>Trustworthiness</h4> | |
| <p class="card-caption">Verification of the LLM-as-a-Checker pipeline demonstrating high | |
| alignment | |
| with human expert judgments, and it is stable across multiple runs.</p> | |
| </div> | |
| </div> | |
| <button class="carousel-btn carousel-next" aria-label="Next"> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="m9 18 6-6-6-6" /> | |
| </svg> | |
| </button> | |
| </div> | |
| <!-- Carousel Dots --> | |
| <div class="carousel-dots" id="results-dots"></div> | |
| </section> | |
| <!-- 3. Scaling Analysis Section --> | |
| <section id="scaling" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <line x1="12" x2="12" y1="20" y2="10" /> | |
| <line x1="18" x2="18" y1="20" y2="4" /> | |
| <line x1="6" x2="6" y1="20" y2="16" /> | |
| </svg> | |
| Scaling Analysis | |
| </h2> | |
| <p>Explore how model performance scales with interaction turns, token usage, and inference cost.</p> | |
| </div> | |
| <div class="dimension-toggle"> | |
| <button class="dim-btn active" data-dim="turn">Turns</button> | |
| <button class="dim-btn" data-dim="token">Tokens</button> | |
| <button class="dim-btn" data-dim="cost">Cost</button> | |
| </div> | |
| <div id="scaling-legend" class="shared-legend"></div> | |
| <div class="charts-grid three-col"> | |
| <div class="chart-card"> | |
| <h3>MIMIC</h3> | |
| <div id="scaling-mimic" class="chart-container"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>10-K</h3> | |
| <div id="scaling-10k" class="chart-container"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>GLOBEM</h3> | |
| <div id="scaling-globem" class="chart-container"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">LLMs extract more accurate insights from delaying commitment, and they | |
| concentrate reasoning into a small number of highly valuable late-stage interactions. These targeted | |
| interactions are built upon longer early exploration.</p> | |
| </section> | |
| <!-- 2. Ranking Comparison Section --> | |
| <section id="ranking" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M6 9H4.5a2.5 2.5 0 0 1 0-5H6" /> | |
| <path d="M18 9h1.5a2.5 2.5 0 0 0 0-5H18" /> | |
| <path d="M4 22h16" /> | |
| <path d="M10 14.66V17c0 .55-.47.98-.97 1.21C7.85 18.75 7 20.24 7 22" /> | |
| <path d="M14 14.66V17c0 .55.47.98.97 1.21C16.15 18.75 17 20.24 17 22" /> | |
| <path d="M18 2H6v7a6 6 0 0 0 12 0V2Z" /> | |
| </svg> | |
| Novelty vs Accuracy | |
| </h2> | |
| <p> | |
| Novelty (Bradley-Terry) vs Accuracy ranking | |
| <br> | |
| ● = Novelty, ◇ = Accuracy. | |
| <br> | |
| <span class="model-badge proprietary">Purple = Proprietary</span> | |
| <span class="model-badge opensource">Green = Open-source</span> | |
| </p> | |
| </div> | |
| <div class="dimension-toggle"> | |
| <button class="dim-btn ranking-dim active" data-mode="novelty">Sort by Novelty</button> | |
| <button class="dim-btn ranking-dim" data-mode="accuracy">Sort by Accuracy</button> | |
| </div> | |
| <div class="charts-grid three-col"> | |
| <div class="chart-card"> | |
| <h3>MIMIC</h3> | |
| <div id="ranking-mimic" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>10-K</h3> | |
| <div id="ranking-10k" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>GLOBEM</h3> | |
| <div id="ranking-globem" class="chart-container-tall"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">The ranking induced by novel insight usefulness closely aligns with the | |
| ranking based on checklist accuracy. Differences between the two rankings are small, especially among | |
| the top-performing models.</p> | |
| </section> | |
| <!-- 3. Turn Distribution Section --> | |
| <section id="turn" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="M21 12a9 9 0 1 1-9-9c2.52 0 4.93 1 6.74 2.74L21 8" /> | |
| <path d="M21 3v5h-5" /> | |
| </svg> | |
| Turn Distribution | |
| </h2> | |
| <p>Analyze the distribution of interaction turns across different models and datasets.</p> | |
| </div> | |
| <div class="charts-grid three-col"> | |
| <div class="chart-card"> | |
| <h3>MIMIC</h3> | |
| <div id="turn-mimic" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>10-K</h3> | |
| <div id="turn-10k" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>GLOBEM</h3> | |
| <div id="turn-globem" class="chart-container-tall"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">Stronger models tend to explore for more rounds without external prompting. | |
| Knowledge-intensive databases such as 10-K and MIMIC induce more interaction rounds than signal-based | |
| datasets such as GLOBEM, and the resulting distributions are also more uniform.</p> | |
| </section> | |
| <!-- 4. Entropy Analysis Section --> | |
| <section id="entropy" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <circle cx="7.5" cy="7.5" r="1.5" /> | |
| <circle cx="18.5" cy="5.5" r="1.5" /> | |
| <circle cx="11.5" cy="11.5" r="1.5" /> | |
| <circle cx="7.5" cy="16.5" r="1.5" /> | |
| <circle cx="17.5" cy="14.5" r="1.5" /> | |
| </svg> | |
| Exploration Pattern | |
| </h2> | |
| <p>Scatter plot showing Access Entropy vs Coverage by model. Opacity represents accuracy. Higher entropy | |
| = more uniform access; Higher coverage = more fields explored.</p> | |
| </div> | |
| <div class="dimension-toggle"> | |
| <button class="toggle-btn active" data-entropy-scenario="10k">10-K</button> | |
| <button class="toggle-btn" data-entropy-scenario="mimic">MIMIC</button> | |
| </div> | |
| <div class="charts-grid three-col"> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-0-title">GPT-5.2</h3> | |
| <div id="entropy-model-0" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-1-title">Claude-4.5-Sonnet</h3> | |
| <div id="entropy-model-1" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-2-title">Gemini-3-Flash</h3> | |
| <div id="entropy-model-2" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-3-title">GLM-4.6</h3> | |
| <div id="entropy-model-3" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-4-title">Qwen3-Next-80B-A3B</h3> | |
| <div id="entropy-model-4" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3 id="entropy-model-5-title">DeepSeek-V3.2</h3> | |
| <div id="entropy-model-5" class="chart-container-tall"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">Advanced LLMs tend to operate in a balanced exploration regime that combines | |
| adequate coverage with focused access. Such a regime is consistently observed across different | |
| scenarios.</p> | |
| </section> | |
| <!-- 5. Error Analysis Section --> | |
| <section id="error" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <path d="m21.73 18-8-14a2 2 0 0 0-3.48 0l-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.73-3Z" /> | |
| <line x1="12" x2="12" y1="9" y2="13" /> | |
| <line x1="12" x2="12.01" y1="17" y2="17" /> | |
| </svg> | |
| Error Analysis | |
| </h2> | |
| <p>Breakdown of error types encountered during agent interactions, grouped by main categories.</p> | |
| </div> | |
| <div class="charts-grid single"> | |
| <div class="chart-card wide"> | |
| <div id="error-chart" class="chart-container-double"></div> | |
| </div> | |
| </div> | |
| <p class="section-description">Our findings revealed that 58% of errors stemmed from insufficient | |
| exploration, both in terms of breadth and depth. This imbalance in exploration often leads to suboptimal | |
| results, regardless of the model’s overall capability. | |
| Additionally, around 40% of the errors were attributed to other factors. For more powerful models, | |
| over-reasoning was common, where the model made assumptions not fully supported by the data. In other | |
| cases, models misinterpreted the insights, such as mistaking a downward trend for an upward one. Less | |
| capable models, on the other hand, tended to make more fundamental errors, such as repeatedly debugging | |
| or struggling with missing data, which could disrupt the overall coherence of the analysis.</p> | |
| </section> | |
| <!-- 6. Probing Results Section --> | |
| <section id="probing" class="section visible"> | |
| <div class="section-header"> | |
| <h2> | |
| <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" | |
| stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> | |
| <circle cx="11" cy="11" r="8" /> | |
| <path d="m21 21-4.3-4.3" /> | |
| </svg> | |
| Self-Termination | |
| </h2> | |
| <p>Analyze the willingness of models to terminate their own analysis.</p> | |
| </div> | |
| <div id="probing-legend" class="shared-legend"></div> | |
| <div class="charts-grid three-col"> | |
| <div class="chart-card"> | |
| <h3>MIMIC</h3> | |
| <div id="probing-mimic" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>GLOBEM</h3> | |
| <div id="probing-globem" class="chart-container-tall"></div> | |
| </div> | |
| <div class="chart-card"> | |
| <h3>10-K</h3> | |
| <div id="probing-10k" class="chart-container-tall"></div> | |
| </div> | |
| </div> | |
| <p class="section-description"> Clear differences emerge across model generations. Qwen3 and Qwen3-Next | |
| exhibit a consistently increasing probability, indicating growing confidence that a complete report can | |
| be produced as more information is accumulated, whereas the Qwen2.5 series shows pronounced fluctuations | |
| and remains uncertain about whether exploration can be terminated at the current step. Moreover, | |
| Qwen3-Next maintains higher confidence with lower variance throughout, suggesting that it has more | |
| confidence that exploration is progressing towards a more comprehensive and deeper report.</p> | |
| </section> | |
| </main> | |
| <!-- Footer --> | |
| <footer class="footer"> | |
| <p>DDR-Bench © 2026 | King's College London · Tencent · The Alan Turing Institute</p> | |
| </footer> | |
| <!-- Carousel Script --> | |
| <script> | |
| document.addEventListener('DOMContentLoaded', function () { | |
| const track = document.getElementById('results-carousel'); | |
| const dotsContainer = document.getElementById('results-dots'); | |
| const prevBtn = document.querySelector('.carousel-prev'); | |
| const nextBtn = document.querySelector('.carousel-next'); | |
| if (!track) return; | |
| const cards = Array.from(track.querySelectorAll('.carousel-card')); | |
| const cardCount = cards.length; | |
| let currentIndex = 0; | |
| let hasStartedAutoPlay = false; | |
| // Create dots | |
| for (let i = 0; i < cardCount; i++) { | |
| const dot = document.createElement('button'); | |
| dot.className = 'carousel-dot' + (i === 0 ? ' active' : ''); | |
| dot.setAttribute('aria-label', `Go to slide ${i + 1}`); | |
| dot.addEventListener('click', () => goToSlide(i)); | |
| dotsContainer.appendChild(dot); | |
| } | |
| const dots = dotsContainer.querySelectorAll('.carousel-dot'); | |
| function updateCarousel() { | |
| // Card is 66.666% width, calculate offset to center current card | |
| const cardWidth = track.offsetWidth * 0.66666; | |
| const gap = 32; // --space-lg | |
| const offset = (track.offsetWidth - cardWidth) / 2 - currentIndex * (cardWidth + gap); | |
| track.style.transform = `translateX(${offset}px)`; | |
| // Update card states | |
| cards.forEach((card, i) => { | |
| card.classList.remove('active', 'side'); | |
| if (i === currentIndex) { | |
| card.classList.add('active'); | |
| } else { | |
| card.classList.add('side'); | |
| } | |
| }); | |
| // Update dots | |
| dots.forEach((dot, i) => { | |
| dot.classList.toggle('active', i === currentIndex); | |
| }); | |
| } | |
| function goToSlide(index) { | |
| // Infinite loop | |
| if (index < 0) { | |
| currentIndex = cardCount - 1; | |
| } else if (index >= cardCount) { | |
| currentIndex = 0; | |
| } else { | |
| currentIndex = index; | |
| } | |
| updateCarousel(); | |
| } | |
| function nextSlide() { goToSlide(currentIndex + 1); } | |
| function prevSlide() { goToSlide(currentIndex - 1); } | |
| // Navigation buttons | |
| prevBtn.addEventListener('click', prevSlide); | |
| nextBtn.addEventListener('click', nextSlide); | |
| // Keyboard navigation | |
| document.addEventListener('keydown', (e) => { | |
| if (e.key === 'ArrowLeft') prevSlide(); | |
| if (e.key === 'ArrowRight') nextSlide(); | |
| }); | |
| // Touch/swipe support | |
| let touchStartX = 0; | |
| track.addEventListener('touchstart', (e) => { | |
| touchStartX = e.changedTouches[0].screenX; | |
| }, { passive: true }); | |
| track.addEventListener('touchend', (e) => { | |
| const diff = touchStartX - e.changedTouches[0].screenX; | |
| if (Math.abs(diff) > 50) { | |
| if (diff > 0) nextSlide(); | |
| else prevSlide(); | |
| } | |
| }, { passive: true }); | |
| // Auto Play Logic | |
| let autoPlayInterval; | |
| const AUTO_PLAY_DELAY = 5000; | |
| function startAutoPlay() { | |
| stopAutoPlay(); // Clear existing to specify | |
| autoPlayInterval = setInterval(nextSlide, AUTO_PLAY_DELAY); | |
| } | |
| function stopAutoPlay() { | |
| if (autoPlayInterval) clearInterval(autoPlayInterval); | |
| } | |
| // Pause on hover | |
| const carouselWrapper = document.querySelector('.carousel-wrapper'); | |
| if (carouselWrapper) { | |
| carouselWrapper.addEventListener('mouseenter', stopAutoPlay); | |
| carouselWrapper.addEventListener('mouseleave', () => { | |
| if (hasStartedAutoPlay) startAutoPlay(); | |
| }); | |
| } | |
| // Initialize carousel position (but don't auto-play yet) | |
| updateCarousel(); | |
| // Use Intersection Observer to start auto-play only when visible | |
| const resultsSection = document.getElementById('results'); | |
| if (resultsSection) { | |
| const carouselObserver = new IntersectionObserver((entries) => { | |
| entries.forEach(entry => { | |
| if (entry.isIntersecting) { | |
| if (!hasStartedAutoPlay) { | |
| hasStartedAutoPlay = true; | |
| } | |
| startAutoPlay(); | |
| } else { | |
| stopAutoPlay(); | |
| } | |
| }); | |
| }, { | |
| threshold: 0.3 // Start when 30% visible | |
| }); | |
| carouselObserver.observe(resultsSection); | |
| } | |
| }); | |
| </script> | |
| <!-- Scroll Hint Script --> | |
| <script> | |
| document.addEventListener('DOMContentLoaded', function () { | |
| const chatWindow = document.getElementById('chat-window'); | |
| const scrollHint = document.getElementById('scroll-hint'); | |
| if (chatWindow && scrollHint) { | |
| // Hide hint when user scrolls | |
| chatWindow.addEventListener('scroll', function () { | |
| if (chatWindow.scrollTop > 50) { | |
| scrollHint.classList.add('hidden'); | |
| } | |
| }); | |
| // Show hint initially after a short delay | |
| setTimeout(() => { | |
| scrollHint.style.opacity = '1'; | |
| }, 1000); | |
| } | |
| }); | |
| </script> | |
| </body> | |
| </html> |