Spaces:
Sleeping
Sleeping
Deploy ELBench-live dashboard
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +1 -0
- .gitignore +24 -0
- README.md +153 -7
- README.zh-CN.md +100 -0
- configs/app.yaml +27 -0
- configs/basic_education.yaml +71 -0
- configs/field_mappings.yaml +98 -0
- configs/file_registry.yaml +241 -0
- configs/gpt54_guided_FIXED.yaml +213 -0
- configs/gpt54_guided_SMOKE.yaml +207 -0
- configs/judges.yaml +35 -0
- configs/models.yaml +473 -0
- configs/module_registry.yaml +17 -0
- configs/providers.yaml +21 -0
- dashboard/README.md +57 -0
- dashboard/app.py +1405 -0
- docs/ARCHITECTURE.md +297 -0
- docs/ARCHITECTURE.zh-CN.md +274 -0
- docs/DASHBOARD_TARGETS.md +187 -0
- docs/RESULT_STORAGE_SPEC.zh-CN.md +148 -0
- docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md +298 -0
- docs/benchmark_status_2026-05-25.md +97 -0
- pyproject.toml +58 -0
- requirements.txt +24 -0
- scripts/build_stage_report.py +423 -0
- scripts/repair_satas_reference_and_rejudge.py +293 -0
- scripts/run_benchmark.py +14 -0
- src/elbench/__init__.py +1 -0
- src/elbench/basic_education_runtime/__init__.py +2 -0
- src/elbench/basic_education_runtime/agent.py +85 -0
- src/elbench/basic_education_runtime/assets/__init__.py +1 -0
- src/elbench/basic_education_runtime/assets/fonts/__init__.py +1 -0
- src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf +3 -0
- src/elbench/basic_education_runtime/cli/draw/__init__.py +35 -0
- src/elbench/basic_education_runtime/cli/eval/__init__.py +147 -0
- src/elbench/basic_education_runtime/cli/export/__init__.py +19 -0
- src/elbench/basic_education_runtime/cli/export/const/label_studio.py +89 -0
- src/elbench/basic_education_runtime/cli/export/exporter/json_.py +110 -0
- src/elbench/basic_education_runtime/cli/export/exporter/label_studio_.py +19 -0
- src/elbench/basic_education_runtime/cli/export/json_.py +50 -0
- src/elbench/basic_education_runtime/cli/export/label_studio.py +61 -0
- src/elbench/basic_education_runtime/cli/generate/__init__.py +26 -0
- src/elbench/basic_education_runtime/cli/main.py +46 -0
- src/elbench/basic_education_runtime/cli/visualize/__init__.py +158 -0
- src/elbench/basic_education_runtime/config.py +42 -0
- src/elbench/basic_education_runtime/directions.py +78 -0
- src/elbench/basic_education_runtime/entity.py +249 -0
- src/elbench/basic_education_runtime/evaluation.py +190 -0
- src/elbench/basic_education_runtime/langchain_compat.py +12 -0
- src/elbench/basic_education_runtime/model.py +88 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
.DS_Store
|
| 3 |
+
*.pyc
|
| 4 |
+
*.pyo
|
| 5 |
+
*.pyd
|
| 6 |
+
.pytest_cache/
|
| 7 |
+
.mypy_cache/
|
| 8 |
+
.ruff_cache/
|
| 9 |
+
.DS_Store
|
| 10 |
+
pytest-cache-files-*/
|
| 11 |
+
.conda/
|
| 12 |
+
.venv/
|
| 13 |
+
venv/
|
| 14 |
+
dist/
|
| 15 |
+
build/
|
| 16 |
+
*.egg-info/
|
| 17 |
+
outputs/
|
| 18 |
+
key.txt
|
| 19 |
+
tmp/
|
| 20 |
+
tmp_test_artifacts/
|
| 21 |
+
results/
|
| 22 |
+
|
| 23 |
+
# Local secrets
|
| 24 |
+
.env
|
README.md
CHANGED
|
@@ -1,13 +1,159 @@
|
|
| 1 |
---
|
| 2 |
-
title: ELBench
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version:
|
| 9 |
-
app_file: app.py
|
|
|
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: ELBench-live
|
| 3 |
+
emoji: "📚"
|
| 4 |
+
colorFrom: blue
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.4.0
|
| 8 |
+
python_version: "3.12"
|
| 9 |
+
app_file: dashboard/app.py
|
| 10 |
+
suggested_hardware: cpu-basic
|
| 11 |
pinned: false
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# ELBench / ELBench-live
|
| 15 |
+
|
| 16 |
+
[](./README.md)
|
| 17 |
+
[](./README.zh-CN.md)
|
| 18 |
+
|
| 19 |
+
📄 [**Paper (arXiv:2608.09548)**](https://arxiv.org/abs/2608.09548) · 💻 [**GitHub (ZeroLoss-Lab/ELBench)**](https://github.com/ZeroLoss-Lab/ELBench) · 🏆 [**AI for Education Leaderboard**](https://benchmarks.ai-for-education.org/) · 🗂️ [**Benchmark Dataset**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) · 📊 [**Results Dataset**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)
|
| 20 |
+
|
| 21 |
+
**ELBench** is a multi-dimensional benchmark for education-facing large language
|
| 22 |
+
models. It evaluates a model across four modules — general capability, safety
|
| 23 |
+
and trustworthiness, basic education, and high-level educational cultivation —
|
| 24 |
+
under one consistent, config-driven evaluation framework.
|
| 25 |
+
|
| 26 |
+
The benchmark covers **2,939 items** across the four modules and is built so a
|
| 27 |
+
single aggregate score never hides where a model actually succeeds or fails.
|
| 28 |
+
|
| 29 |
+
## Leaderboard
|
| 30 |
+
|
| 31 |
+
Overall scores are reported on a 0–100 scale with a bootstrap 95% confidence
|
| 32 |
+
interval over items (10,000 resamples). The top models are statistically close:
|
| 33 |
+
the spread among the first six is under two points, and several share rank 4.
|
| 34 |
+
|
| 35 |
+
| Rank | Model | Overall [95% CI] | General | Safety | Basic Ed. | High-Level |
|
| 36 |
+
|:----:|:------|:----------------:|:-------:|:------:|:---------:|:----------:|
|
| 37 |
+
| 1 | DeepSeek-V4-Flash | **83.7** [82, 85] | 88.5 | 89.7 | 84.9 | 71.5 |
|
| 38 |
+
| 2 | Gemini-3.5-Flash | **83.4** [82, 84] | 93.4 | 70.2 | 94.6 | 75.3 |
|
| 39 |
+
| 3 | Doubao-Seed-2.0-Pro | **83.2** [82, 84] | 86.6 | 83.0 | 89.7 | 73.5 |
|
| 40 |
+
| 4 | Claude-Opus-4.8 | **83.1** [82, 84] | 91.9 | 78.5 | 92.7 | 69.5 |
|
| 41 |
+
| 4 | GPT-5.4 | **83.1** [82, 84] | 86.4 | 76.6 | 94.4 | 75.1 |
|
| 42 |
+
| 4 | DeepSeek-V4-Pro | **83.1** [82, 84] | 88.2 | 86.1 | 88.5 | 69.5 |
|
| 43 |
+
| 7 | GLM-5.1 | **81.7** [80, 84] | 83.2 | 89.5 | 79.2 | 75.0 |
|
| 44 |
+
| 8 | Safe-InnoSpark | **77.0** [76, 78] | 68.0 | 87.6 | 87.3 | 65.2 |
|
| 45 |
+
| 9 | InnoSpark-235B | **76.4** [75, 78] | 74.2 | 78.0 | 87.5 | 65.9 |
|
| 46 |
+
|
| 47 |
+
Per-module and per-task leaderboards, confidence intervals, and the
|
| 48 |
+
judge-reliability audit are released as the
|
| 49 |
+
[ZeroLoss-Lab/ELBench-results](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)
|
| 50 |
+
dataset on HuggingFace.
|
| 51 |
+
|
| 52 |
+
## Modules
|
| 53 |
+
|
| 54 |
+
| Module | Items | What it measures |
|
| 55 |
+
|:-------|:-----:|:-----------------|
|
| 56 |
+
| General Capability (`通用模型`) | 894 | Knowledge, reasoning, instruction following, and math (MMLU-Pro, C-Eval, IFEval, MATH-500, AIME samples) |
|
| 57 |
+
| Safety & Trustworthiness (`安全可信`) | 1,000 | Harmful-request refusal, helpfulness on benign requests, and adversarial robustness |
|
| 58 |
+
| Basic Education (`基本教育`) | 45 | Multi-turn teaching: knowledge explanation, situated question design, cross-subject lesson plans, guided problem solving |
|
| 59 |
+
| High-Level Educational Cultivation (`高阶育人`) | 1,000 | Open-ended cultivation of higher-order educational goals |
|
| 60 |
+
|
| 61 |
+
Objective tasks are graded by rule or reference matching; subjective and
|
| 62 |
+
open-ended tasks use `LLM-as-a-Judge` with a validated judge panel. The
|
| 63 |
+
judge-agreement audit (quadratic weighted Cohen's κ against a human gold set)
|
| 64 |
+
is in the [results dataset](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results).
|
| 65 |
+
|
| 66 |
+
## Installation
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
pip install -e . # core framework
|
| 70 |
+
pip install -e .[basic-education] # + Basic Education multi-turn runtime
|
| 71 |
+
pip install -e .[basic-education,dev] # + contributor tooling
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
## Quick Start
|
| 75 |
+
|
| 76 |
+
```bash
|
| 77 |
+
# Inspect the registered benchmark files
|
| 78 |
+
python scripts/run_benchmark.py inspect
|
| 79 |
+
|
| 80 |
+
# Local smoke test against the mock provider (no API keys needed)
|
| 81 |
+
python scripts/run_benchmark.py run \
|
| 82 |
+
--model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
After an editable install the `elbench` entrypoint is also available:
|
| 86 |
+
|
| 87 |
+
```bash
|
| 88 |
+
elbench inspect
|
| 89 |
+
elbench run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
|
| 90 |
+
```
|
| 91 |
+
|
| 92 |
+
To evaluate a real model, add it to `configs/models.yaml` and run a module:
|
| 93 |
+
|
| 94 |
+
```bash
|
| 95 |
+
python scripts/run_benchmark.py run \
|
| 96 |
+
--model-id <your_model_id> --module 基本教育 --run-id basic-education-run
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
`基本教育` (Basic Education) requires a real API-backed model config; the
|
| 100 |
+
`mock.*` models are for single-turn pipeline smoke checks only.
|
| 101 |
+
|
| 102 |
+
## Hosted runner
|
| 103 |
+
|
| 104 |
+
`dashboard/app.py` is a Gradio app for a visitor who brings an OpenRouter key.
|
| 105 |
+
It calls `BenchmarkRunner` and `JudgeRouter`. Prompts, retries, and scores are
|
| 106 |
+
not reimplemented in the UI. The estimate states the cost range and the ways a
|
| 107 |
+
subset, a `max_samples` prefix, or an unlisted model diverges from a published
|
| 108 |
+
run. The YAML block at the top of this file is the Hugging Face Space config.
|
| 109 |
+
CPU Basic has no hourly price, but creating the Gradio Space requires a paid plan.
|
| 110 |
+
Details are in [dashboard/README.md](dashboard/README.md).
|
| 111 |
+
|
| 112 |
+
```bash
|
| 113 |
+
pip install -e ".[basic-education,dashboard]"
|
| 114 |
+
python dashboard/app.py
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
## Data on HuggingFace
|
| 118 |
+
|
| 119 |
+
ELBench is released as two HuggingFace datasets:
|
| 120 |
+
|
| 121 |
+
- [**ZeroLoss-Lab/ELBench**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) — the benchmark items (2,939 across the four modules).
|
| 122 |
+
- [**ZeroLoss-Lab/ELBench-results**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) — the full evaluation results, including per-sample detail.
|
| 123 |
+
|
| 124 |
+
This repository holds the evaluation framework and code only; the benchmark
|
| 125 |
+
items and results live in the two datasets above. Run artifacts
|
| 126 |
+
(`outputs/raw_responses/`, `judged_results/`, `logs/`) are runtime-only and not
|
| 127 |
+
committed.
|
| 128 |
+
|
| 129 |
+
## Documentation
|
| 130 |
+
|
| 131 |
+
- [Architecture](docs/ARCHITECTURE.md) — registry, provider abstraction,
|
| 132 |
+
execution model, and how to extend the framework
|
| 133 |
+
- [Team maintenance guide](docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md) — what each
|
| 134 |
+
directory is for and what should or should not be committed
|
| 135 |
+
|
| 136 |
+
## Tests
|
| 137 |
+
|
| 138 |
+
```bash
|
| 139 |
+
python -m unittest \
|
| 140 |
+
tests.test_basic_education_bridge tests.test_registry_and_loaders \
|
| 141 |
+
tests.test_judges tests.test_response_format_retry
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
## 🔬 Citation
|
| 145 |
+
|
| 146 |
+
If you use this benchmark in your research, please cite:
|
| 147 |
+
|
| 148 |
+
```bibtex
|
| 149 |
+
@article{jiang2026elbench,
|
| 150 |
+
title={ELBench: A Multi-Dimensional Benchmark for Education-Facing Large Language Models},
|
| 151 |
+
author={Jiang, Yilin and Zhu, Xiaorong and Tan, Fei and Zhang, Zicheng and Huang, Kaiyi and Yu, Yang and Fei, Zexuan and Luo, Yiming and Li, Keqian and Hao, Hao and others},
|
| 152 |
+
journal={arXiv preprint arXiv:2608.09548},
|
| 153 |
+
year={2026}
|
| 154 |
+
}
|
| 155 |
+
```
|
| 156 |
+
|
| 157 |
+
## License
|
| 158 |
+
|
| 159 |
+
Add your preferred license before publishing.
|
README.zh-CN.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench
|
| 2 |
+
|
| 3 |
+
[](./README.md)
|
| 4 |
+
[](./README.zh-CN.md)
|
| 5 |
+
|
| 6 |
+
**ELBench** 是一个面向教育大模型的多维度评测基准。它从四个模块评估模型——通用能力、安全可信、基本教育、高阶育人——并在统一的、配置驱动的评测框架下运行。
|
| 7 |
+
|
| 8 |
+
基准共覆盖 **2939 道题**,横跨四个模块;设计上让单一总分无法掩盖模型在各模块上的真实强弱。
|
| 9 |
+
|
| 10 |
+
## 排行榜
|
| 11 |
+
|
| 12 |
+
总分采用 0–100 量纲,并给出基于条目级 bootstrap(10000 次重采样)的 95% 置信区间。头部模型相当接近:前六名分差不到两分,多个模型并列第 4。
|
| 13 |
+
|
| 14 |
+
| 排名 | 模型 | 总分 [95% CI] | 通用 | 安全 | 基本教育 | 高阶育人 |
|
| 15 |
+
|:----:|:-----|:------------:|:----:|:----:|:--------:|:--------:|
|
| 16 |
+
| 1 | DeepSeek-V4-Flash | **83.7** [82, 85] | 88.5 | 89.7 | 84.9 | 71.5 |
|
| 17 |
+
| 2 | Gemini-3.5-Flash | **83.4** [82, 84] | 93.4 | 70.2 | 94.6 | 75.3 |
|
| 18 |
+
| 3 | Doubao-Seed-2.0-Pro | **83.2** [82, 84] | 86.6 | 83.0 | 89.7 | 73.5 |
|
| 19 |
+
| 4 | Claude-Opus-4.8 | **83.1** [82, 84] | 91.9 | 78.5 | 92.7 | 69.5 |
|
| 20 |
+
| 4 | GPT-5.4 | **83.1** [82, 84] | 86.4 | 76.6 | 94.4 | 75.1 |
|
| 21 |
+
| 4 | DeepSeek-V4-Pro | **83.1** [82, 84] | 88.2 | 86.1 | 88.5 | 69.5 |
|
| 22 |
+
| 7 | GLM-5.1 | **81.7** [80, 84] | 83.2 | 89.5 | 79.2 | 75.0 |
|
| 23 |
+
| 8 | Safe-InnoSpark | **77.0** [76, 78] | 68.0 | 87.6 | 87.3 | 65.2 |
|
| 24 |
+
| 9 | InnoSpark-235B | **76.4** [75, 78] | 74.2 | 78.0 | 87.5 | 65.9 |
|
| 25 |
+
|
| 26 |
+
各模块、各任务的明细榜、置信区间以及评分可靠性审计,以 HuggingFace 数据集
|
| 27 |
+
[ZeroLoss-Lab/ELBench-results](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) 发布。
|
| 28 |
+
|
| 29 |
+
## 模块
|
| 30 |
+
|
| 31 |
+
| 模块 | 题量 | 评测内容 |
|
| 32 |
+
|:-----|:----:|:---------|
|
| 33 |
+
| 通用能力(`通用模型`) | 894 | 知识、推理、指令遵循与数学(MMLU-Pro、C-Eval、IFEval、MATH-500、AIME 采样) |
|
| 34 |
+
| 安全可信(`安全可信`) | 1000 | 有害请求拒答、良性请求的有用性、对抗鲁棒性 |
|
| 35 |
+
| 基本教育(`基本教育`) | 45 | 多轮教学:知识点讲解、情景化出题、跨学科教案、引导式讲题 |
|
| 36 |
+
| 高阶育人(`高阶育人`) | 1000 | 高阶育人目标的开放式培养 |
|
| 37 |
+
|
| 38 |
+
客观题由规则或参考答案匹配判分;主观和开放式任务采用 `LLM-as-a-Judge`,并配有经过验证的评分模型面板。评分一致性审计(对人工金标的二次加权 Cohen κ)见
|
| 39 |
+
[结果数据集](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)。
|
| 40 |
+
|
| 41 |
+
## 安装
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
pip install -e . # 核心框架
|
| 45 |
+
pip install -e .[basic-education] # + 基本教育多轮运行时
|
| 46 |
+
pip install -e .[basic-education,dev] # + 贡献者工具链
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
## 快速开始
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
# 查看已注册的 benchmark 文件
|
| 53 |
+
python scripts/run_benchmark.py inspect
|
| 54 |
+
|
| 55 |
+
# 用 mock provider 跑本地冒烟测试(无需 API key)
|
| 56 |
+
python scripts/run_benchmark.py run \
|
| 57 |
+
--model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
editable 安装后也可直接用 `elbench` 入口:
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
elbench inspect
|
| 64 |
+
elbench run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
评测真实模型时,先在 `configs/models.yaml` 里登记模型,再按模块运行:
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
python scripts/run_benchmark.py run \
|
| 71 |
+
--model-id <你的模型ID> --module 基本教育 --run-id basic-education-run
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
`基本教育` 需要真实的 API 模型配置;`mock.*` 模型仅用于单轮流水线冒烟检查。
|
| 75 |
+
|
| 76 |
+
## HuggingFace 数据集
|
| 77 |
+
|
| 78 |
+
ELBench 以两个 HuggingFace 数据集发布:
|
| 79 |
+
|
| 80 |
+
- [**ZeroLoss-Lab/ELBench**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) —— 评测题目集(四模块共 2939 题)。
|
| 81 |
+
- [**ZeroLoss-Lab/ELBench-results**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) —— 完整评测结果,含逐样本明细。
|
| 82 |
+
|
| 83 |
+
本仓库只保留评测框架与代码;评测题目与结果都在上面两个数据集中。原始单次运行产物(`outputs/raw_responses/`、`judged_results/`、`logs/`)只在运行时存在,不进版本库。
|
| 84 |
+
|
| 85 |
+
## 文档
|
| 86 |
+
|
| 87 |
+
- [架构说明](docs/ARCHITECTURE.zh-CN.md):注册表、provider 抽象、执行模型,以及如何扩展框架
|
| 88 |
+
- [团队维护指南](docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md):各目录的用途,以及哪些该提交、哪些不该提交
|
| 89 |
+
|
| 90 |
+
## 测试
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
python -m unittest \
|
| 94 |
+
tests.test_basic_education_bridge tests.test_registry_and_loaders \
|
| 95 |
+
tests.test_judges tests.test_response_format_retry
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
## 许可证
|
| 99 |
+
|
| 100 |
+
发布前请补充你选择的许可证。
|
configs/app.yaml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
project_name: ELBench
|
| 2 |
+
data_root: data/benchmark_root
|
| 3 |
+
output_root: outputs
|
| 4 |
+
default_run:
|
| 5 |
+
max_concurrency: 100
|
| 6 |
+
checkpoint_interval: 1
|
| 7 |
+
log_every_n: 50
|
| 8 |
+
judge_enabled: true
|
| 9 |
+
summary_enabled: true
|
| 10 |
+
resume: true
|
| 11 |
+
retry:
|
| 12 |
+
max_attempts: 3
|
| 13 |
+
initial_delay_seconds: 1.0
|
| 14 |
+
max_delay_seconds: 20.0
|
| 15 |
+
backoff_multiplier: 2.0
|
| 16 |
+
jitter_ratio: 0.2
|
| 17 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 18 |
+
provider_rate_limits:
|
| 19 |
+
default:
|
| 20 |
+
max_concurrency: 100
|
| 21 |
+
benchmark_modules:
|
| 22 |
+
active:
|
| 23 |
+
- 安全可信
|
| 24 |
+
- 高阶育人
|
| 25 |
+
- 通用模型
|
| 26 |
+
- 基本教育
|
| 27 |
+
reserved: []
|
configs/basic_education.yaml
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
enabled: true
|
| 2 |
+
module_name: 基本教育
|
| 3 |
+
runtime_python: python
|
| 4 |
+
runtime_cli_module: elbench.basic_education_runtime.cli.main
|
| 5 |
+
command_timeout_seconds: 14400
|
| 6 |
+
continue_on_error: true
|
| 7 |
+
workspace_root: outputs/basic_education
|
| 8 |
+
test_endpoint_model_id: qwen-test
|
| 9 |
+
evaluator_model_id: null
|
| 10 |
+
default_pass_threshold: 3.0
|
| 11 |
+
scenarios:
|
| 12 |
+
- scenario_id: knowledge_point_explanation
|
| 13 |
+
source_file: knowledge.yaml
|
| 14 |
+
subset: 知识点讲解
|
| 15 |
+
task: basic_knowledge_point_explanation
|
| 16 |
+
template_path: 基本教育/知识点讲解/knowledge.yaml
|
| 17 |
+
expected_tasks: 10
|
| 18 |
+
multi_turn: false
|
| 19 |
+
dimension: 知识点讲解
|
| 20 |
+
target_model_keys: [4o_teacher]
|
| 21 |
+
test_endpoint_model_keys: []
|
| 22 |
+
judge_model_keys: [evaler]
|
| 23 |
+
pass_threshold: 3.0
|
| 24 |
+
metadata:
|
| 25 |
+
topic: basic_education
|
| 26 |
+
scene: knowledge
|
| 27 |
+
- scenario_id: contextualized_question_generation
|
| 28 |
+
source_file: question.yaml
|
| 29 |
+
subset: 情景化出题
|
| 30 |
+
task: basic_contextualized_question_generation
|
| 31 |
+
template_path: 基本教育/情景化出题/question.yaml
|
| 32 |
+
expected_tasks: 10
|
| 33 |
+
multi_turn: false
|
| 34 |
+
dimension: 情景化出题
|
| 35 |
+
target_model_keys: [test_model]
|
| 36 |
+
test_endpoint_model_keys: []
|
| 37 |
+
judge_model_keys: [judge_model]
|
| 38 |
+
pass_threshold: 3.0
|
| 39 |
+
metadata:
|
| 40 |
+
topic: basic_education
|
| 41 |
+
scene: question
|
| 42 |
+
- scenario_id: interdisciplinary_lesson_plan_generation
|
| 43 |
+
source_file: cross.yaml
|
| 44 |
+
subset: 跨学科教案生成
|
| 45 |
+
task: basic_interdisciplinary_lesson_plan_generation
|
| 46 |
+
template_path: 基本教育/跨学科教案生成/cross.yaml
|
| 47 |
+
expected_tasks: 10
|
| 48 |
+
multi_turn: false
|
| 49 |
+
dimension: 跨学科教案生成
|
| 50 |
+
target_model_keys: [test_model]
|
| 51 |
+
test_endpoint_model_keys: []
|
| 52 |
+
judge_model_keys: [judge_model]
|
| 53 |
+
pass_threshold: 3.0
|
| 54 |
+
metadata:
|
| 55 |
+
topic: basic_education
|
| 56 |
+
scene: cross
|
| 57 |
+
- scenario_id: guided_problem_solving_teaching
|
| 58 |
+
source_file: config_guided_task.yaml
|
| 59 |
+
subset: 引导式讲题
|
| 60 |
+
task: basic_guided_problem_solving_teaching
|
| 61 |
+
template_path: 基本教育/引导式讲题/config_guided_task.yaml
|
| 62 |
+
expected_tasks: 15
|
| 63 |
+
multi_turn: true
|
| 64 |
+
dimension: 引导式讲题
|
| 65 |
+
target_model_keys: [teacher]
|
| 66 |
+
test_endpoint_model_keys: [stu]
|
| 67 |
+
judge_model_keys: [eval]
|
| 68 |
+
pass_threshold: 3.0
|
| 69 |
+
metadata:
|
| 70 |
+
topic: basic_education
|
| 71 |
+
scene: guided
|
configs/field_mappings.yaml
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mappings:
|
| 2 |
+
safety_refusal_jsonl:
|
| 3 |
+
sample_id_fields: []
|
| 4 |
+
prompt_fields: [question]
|
| 5 |
+
dimension_fields: []
|
| 6 |
+
reference_fields: [label]
|
| 7 |
+
metadata_fields: [label, subcategory]
|
| 8 |
+
safety_guidance_jsonl:
|
| 9 |
+
sample_id_fields: []
|
| 10 |
+
prompt_fields: [question]
|
| 11 |
+
dimension_fields: []
|
| 12 |
+
reference_fields: [question_type, category]
|
| 13 |
+
metadata_fields: [question_type, category, subcategory]
|
| 14 |
+
safety_answer_jsonl:
|
| 15 |
+
sample_id_fields: []
|
| 16 |
+
prompt_fields: [question]
|
| 17 |
+
dimension_fields: []
|
| 18 |
+
reference_fields: [label]
|
| 19 |
+
metadata_fields: [label, subcategory]
|
| 20 |
+
teaching_harm_xlsx:
|
| 21 |
+
sample_id_fields: [ID]
|
| 22 |
+
dimension_fields: []
|
| 23 |
+
reference_fields: [Answer]
|
| 24 |
+
metadata_fields: [Question_Chinese, Question_English, Answer]
|
| 25 |
+
prompt_template: |
|
| 26 |
+
[Question_Chinese]
|
| 27 |
+
{{ Question_Chinese }}
|
| 28 |
+
|
| 29 |
+
[Question_English]
|
| 30 |
+
{{ Question_English }}
|
| 31 |
+
adversarial_safety_xlsx:
|
| 32 |
+
sample_id_fields: [ID]
|
| 33 |
+
dimension_fields: []
|
| 34 |
+
reference_fields: []
|
| 35 |
+
metadata_fields:
|
| 36 |
+
[Teacher_Prompt_EN, Student_Statement_EN, Teacher_Prompt_CN, Student_Statement_CN]
|
| 37 |
+
prompt_template: |
|
| 38 |
+
[Teacher_Prompt_CN]
|
| 39 |
+
{{ Teacher_Prompt_CN }}
|
| 40 |
+
|
| 41 |
+
[Student_Statement_CN]
|
| 42 |
+
{{ Student_Statement_CN }}
|
| 43 |
+
|
| 44 |
+
[Teacher_Prompt_EN]
|
| 45 |
+
{{ Teacher_Prompt_EN }}
|
| 46 |
+
|
| 47 |
+
[Student_Statement_EN]
|
| 48 |
+
{{ Student_Statement_EN }}
|
| 49 |
+
highlevel_edu_jsonl:
|
| 50 |
+
sample_id_fields: []
|
| 51 |
+
prompt_fields: [question]
|
| 52 |
+
dimension_fields: []
|
| 53 |
+
dimension_resolver: highlevel_edu
|
| 54 |
+
reference_fields: [Score, Scoring Details, Personalized Feedback]
|
| 55 |
+
metadata_fields:
|
| 56 |
+
[Subject, Level, Question Type, Question, Student's Answer, Score, Scoring Details, Personalized Feedback, _scene]
|
| 57 |
+
highlevel_omni_jsonl:
|
| 58 |
+
sample_id_fields: [id]
|
| 59 |
+
prompt_fields: [question]
|
| 60 |
+
dimension_fields: [field]
|
| 61 |
+
reference_fields: [answer, reasonableness]
|
| 62 |
+
metadata_fields: [type, field, answer, reasonableness]
|
| 63 |
+
basic_education_yaml:
|
| 64 |
+
sample_id_fields: []
|
| 65 |
+
prompt_fields: []
|
| 66 |
+
dimension_fields: []
|
| 67 |
+
reference_fields: []
|
| 68 |
+
metadata_fields: []
|
| 69 |
+
mmlu_pro_jsonl:
|
| 70 |
+
sample_id_fields: [id]
|
| 71 |
+
prompt_fields: []
|
| 72 |
+
dimension_fields: [subset_key]
|
| 73 |
+
reference_fields: [target]
|
| 74 |
+
metadata_fields: [metadata, choices, subset_key]
|
| 75 |
+
ceval_jsonl:
|
| 76 |
+
sample_id_fields: [id]
|
| 77 |
+
prompt_fields: []
|
| 78 |
+
dimension_fields: [metadata.subject]
|
| 79 |
+
reference_fields: [target]
|
| 80 |
+
metadata_fields: [metadata, choices, subset_key]
|
| 81 |
+
ifeval_jsonl:
|
| 82 |
+
sample_id_fields: [id]
|
| 83 |
+
prompt_fields: []
|
| 84 |
+
dimension_fields: []
|
| 85 |
+
reference_fields: [target]
|
| 86 |
+
metadata_fields: [metadata, instruction_id_list, kwargs, key, prompt]
|
| 87 |
+
math_500_jsonl:
|
| 88 |
+
sample_id_fields: [id]
|
| 89 |
+
prompt_fields: []
|
| 90 |
+
dimension_fields: [subset_key]
|
| 91 |
+
reference_fields: [target]
|
| 92 |
+
metadata_fields: [metadata, subset_key]
|
| 93 |
+
aime_jsonl:
|
| 94 |
+
sample_id_fields: [id]
|
| 95 |
+
prompt_fields: []
|
| 96 |
+
dimension_fields: []
|
| 97 |
+
reference_fields: [target]
|
| 98 |
+
metadata_fields: [metadata, subset_key]
|
configs/file_registry.yaml
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
files:
|
| 2 |
+
- canonical_name: 安全拒答.jsonl
|
| 3 |
+
module: 安全可信
|
| 4 |
+
subset: 通用-应拒答
|
| 5 |
+
file_type: jsonl
|
| 6 |
+
task: safety_refusal
|
| 7 |
+
loader_name: jsonl
|
| 8 |
+
mapping_key: safety_refusal_jsonl
|
| 9 |
+
path_hints:
|
| 10 |
+
- 安全可信/通用-应拒答/安全拒答.jsonl
|
| 11 |
+
- 安全拒答.jsonl
|
| 12 |
+
filename_patterns:
|
| 13 |
+
- 安全拒答.jsonl
|
| 14 |
+
aliases: []
|
| 15 |
+
- canonical_name: 安全引导.jsonl
|
| 16 |
+
module: 安全可信
|
| 17 |
+
subset: 通用-引导回答
|
| 18 |
+
file_type: jsonl
|
| 19 |
+
task: safety_guidance
|
| 20 |
+
loader_name: jsonl
|
| 21 |
+
mapping_key: safety_guidance_jsonl
|
| 22 |
+
path_hints:
|
| 23 |
+
- 安全可信/通用-引导回答/安全引导.jsonl
|
| 24 |
+
- 安全引导.jsonl
|
| 25 |
+
filename_patterns:
|
| 26 |
+
- 安全引导.jsonl
|
| 27 |
+
aliases: []
|
| 28 |
+
- canonical_name: 安全回答.jsonl
|
| 29 |
+
module: 安全可信
|
| 30 |
+
subset: 通用-应回答
|
| 31 |
+
file_type: jsonl
|
| 32 |
+
task: safety_answer
|
| 33 |
+
loader_name: jsonl
|
| 34 |
+
mapping_key: safety_answer_jsonl
|
| 35 |
+
path_hints:
|
| 36 |
+
- 安全可信/通用-应回答/安全回答.jsonl
|
| 37 |
+
- 安全回答.jsonl
|
| 38 |
+
filename_patterns:
|
| 39 |
+
- 安全回答.jsonl
|
| 40 |
+
aliases: []
|
| 41 |
+
- canonical_name: SATAs.xlsx
|
| 42 |
+
module: 安全可信
|
| 43 |
+
subset: 教育-教学安全
|
| 44 |
+
file_type: xlsx
|
| 45 |
+
task: teaching_harm
|
| 46 |
+
loader_name: xlsx
|
| 47 |
+
mapping_key: teaching_harm_xlsx
|
| 48 |
+
path_hints:
|
| 49 |
+
- 安全可信/教育-教学安全/SATAs.xlsx
|
| 50 |
+
- SATAs.xlsx
|
| 51 |
+
- SATAs(150).xlsx
|
| 52 |
+
filename_patterns:
|
| 53 |
+
- SATAs.xlsx
|
| 54 |
+
- SATAs*.xlsx
|
| 55 |
+
aliases:
|
| 56 |
+
- SATAs(150).xlsx
|
| 57 |
+
- canonical_name: adversarial_prompts.xlsx
|
| 58 |
+
module: 安全可信
|
| 59 |
+
subset: 教育-教学安全
|
| 60 |
+
file_type: xlsx
|
| 61 |
+
task: adversarial_safety
|
| 62 |
+
loader_name: xlsx
|
| 63 |
+
mapping_key: adversarial_safety_xlsx
|
| 64 |
+
path_hints:
|
| 65 |
+
- 安全可信/教育-教学安全/adversarial_prompts.xlsx
|
| 66 |
+
- adversarial_prompts.xlsx
|
| 67 |
+
- adversarial_prompts(100).xlsx
|
| 68 |
+
filename_patterns:
|
| 69 |
+
- adversarial_prompts.xlsx
|
| 70 |
+
- adversarial_prompts*.xlsx
|
| 71 |
+
aliases:
|
| 72 |
+
- adversarial_prompts(100).xlsx
|
| 73 |
+
- canonical_name: 高阶育人-edu.jsonl
|
| 74 |
+
module: 高阶育人
|
| 75 |
+
subset: edu
|
| 76 |
+
file_type: jsonl
|
| 77 |
+
task: highlevel_edu
|
| 78 |
+
loader_name: jsonl
|
| 79 |
+
mapping_key: highlevel_edu_jsonl
|
| 80 |
+
path_hints:
|
| 81 |
+
- 高阶育人/edu/高阶育人-edu.jsonl
|
| 82 |
+
- 高阶育人-edu.jsonl
|
| 83 |
+
filename_patterns:
|
| 84 |
+
- 高阶育人-edu.jsonl
|
| 85 |
+
aliases: []
|
| 86 |
+
- canonical_name: 高阶育人-omni.jsonl
|
| 87 |
+
module: 高阶育人
|
| 88 |
+
subset: omni
|
| 89 |
+
file_type: jsonl
|
| 90 |
+
task: highlevel_omni
|
| 91 |
+
loader_name: jsonl
|
| 92 |
+
mapping_key: highlevel_omni_jsonl
|
| 93 |
+
path_hints:
|
| 94 |
+
- 高阶育人/omni/高阶育人-omni.jsonl
|
| 95 |
+
- 高阶育人-omni.jsonl
|
| 96 |
+
filename_patterns:
|
| 97 |
+
- 高阶育人-omni.jsonl
|
| 98 |
+
aliases: []
|
| 99 |
+
- canonical_name: knowledge.yaml
|
| 100 |
+
module: 基本教育
|
| 101 |
+
subset: 知识点讲解
|
| 102 |
+
file_type: yaml
|
| 103 |
+
task: basic_knowledge_point_explanation
|
| 104 |
+
loader_name: basic_education_yaml
|
| 105 |
+
mapping_key: basic_education_yaml
|
| 106 |
+
path_hints:
|
| 107 |
+
- 基本教育/知识点讲解/knowledge.yaml
|
| 108 |
+
- knowledge.yaml
|
| 109 |
+
filename_patterns:
|
| 110 |
+
- knowledge.yaml
|
| 111 |
+
aliases: []
|
| 112 |
+
- canonical_name: question.yaml
|
| 113 |
+
module: 基本教育
|
| 114 |
+
subset: 情景化出题
|
| 115 |
+
file_type: yaml
|
| 116 |
+
task: basic_contextualized_question_generation
|
| 117 |
+
loader_name: basic_education_yaml
|
| 118 |
+
mapping_key: basic_education_yaml
|
| 119 |
+
path_hints:
|
| 120 |
+
- 基本教育/情景化出题/question.yaml
|
| 121 |
+
- question.yaml
|
| 122 |
+
filename_patterns:
|
| 123 |
+
- question.yaml
|
| 124 |
+
aliases: []
|
| 125 |
+
- canonical_name: cross.yaml
|
| 126 |
+
module: 基本教育
|
| 127 |
+
subset: 跨学科教案生成
|
| 128 |
+
file_type: yaml
|
| 129 |
+
task: basic_interdisciplinary_lesson_plan_generation
|
| 130 |
+
loader_name: basic_education_yaml
|
| 131 |
+
mapping_key: basic_education_yaml
|
| 132 |
+
path_hints:
|
| 133 |
+
- 基本教育/跨学科教案生成/cross.yaml
|
| 134 |
+
- cross.yaml
|
| 135 |
+
filename_patterns:
|
| 136 |
+
- cross.yaml
|
| 137 |
+
aliases: []
|
| 138 |
+
- canonical_name: config_guided_task.yaml
|
| 139 |
+
module: 基本教育
|
| 140 |
+
subset: 引导式讲题
|
| 141 |
+
file_type: yaml
|
| 142 |
+
task: basic_guided_problem_solving_teaching
|
| 143 |
+
loader_name: basic_education_yaml
|
| 144 |
+
mapping_key: basic_education_yaml
|
| 145 |
+
path_hints:
|
| 146 |
+
- 基本教育/引导式讲题/config_guided_task.yaml
|
| 147 |
+
- config_guided_task.yaml
|
| 148 |
+
filename_patterns:
|
| 149 |
+
- config_guided_task.yaml
|
| 150 |
+
aliases: []
|
| 151 |
+
- canonical_name: mmlu_pro_sampled.jsonl
|
| 152 |
+
module: 通用模型
|
| 153 |
+
subset: mmlu_pro
|
| 154 |
+
file_type: jsonl
|
| 155 |
+
task: mmlu_pro
|
| 156 |
+
loader_name: mmlu_pro_jsonl
|
| 157 |
+
mapping_key: mmlu_pro_jsonl
|
| 158 |
+
path_hints:
|
| 159 |
+
- 通用/mmlu_pro_sampled.jsonl
|
| 160 |
+
- mmlu_pro_sampled.jsonl
|
| 161 |
+
filename_patterns:
|
| 162 |
+
- mmlu_pro_sampled.jsonl
|
| 163 |
+
aliases: []
|
| 164 |
+
- canonical_name: ceval_sampled.jsonl
|
| 165 |
+
module: 通用模型
|
| 166 |
+
subset: ceval
|
| 167 |
+
file_type: jsonl
|
| 168 |
+
task: ceval
|
| 169 |
+
loader_name: ceval_jsonl
|
| 170 |
+
mapping_key: ceval_jsonl
|
| 171 |
+
path_hints:
|
| 172 |
+
- 通用/ceval_sampled.jsonl
|
| 173 |
+
- ceval_sampled.jsonl
|
| 174 |
+
filename_patterns:
|
| 175 |
+
- ceval_sampled.jsonl
|
| 176 |
+
aliases: []
|
| 177 |
+
- canonical_name: ifeval_sampled.jsonl
|
| 178 |
+
module: 通用模型
|
| 179 |
+
subset: ifeval
|
| 180 |
+
file_type: jsonl
|
| 181 |
+
task: ifeval
|
| 182 |
+
loader_name: ifeval_jsonl
|
| 183 |
+
mapping_key: ifeval_jsonl
|
| 184 |
+
path_hints:
|
| 185 |
+
- 通用/ifeval_sampled.jsonl
|
| 186 |
+
- ifeval_sampled.jsonl
|
| 187 |
+
filename_patterns:
|
| 188 |
+
- ifeval_sampled.jsonl
|
| 189 |
+
aliases: []
|
| 190 |
+
- canonical_name: math_500_sampled.jsonl
|
| 191 |
+
module: 通用模型
|
| 192 |
+
subset: math_500
|
| 193 |
+
file_type: jsonl
|
| 194 |
+
task: math_500
|
| 195 |
+
loader_name: math_500_jsonl
|
| 196 |
+
mapping_key: math_500_jsonl
|
| 197 |
+
path_hints:
|
| 198 |
+
- 通用/math_500_sampled.jsonl
|
| 199 |
+
- math_500_sampled.jsonl
|
| 200 |
+
filename_patterns:
|
| 201 |
+
- math_500_sampled.jsonl
|
| 202 |
+
aliases: []
|
| 203 |
+
- canonical_name: aime24_sampled.jsonl
|
| 204 |
+
module: 通用模型
|
| 205 |
+
subset: aime24
|
| 206 |
+
file_type: jsonl
|
| 207 |
+
task: aime24
|
| 208 |
+
loader_name: aime_jsonl
|
| 209 |
+
mapping_key: aime_jsonl
|
| 210 |
+
path_hints:
|
| 211 |
+
- 通用/aime24_sampled.jsonl
|
| 212 |
+
- aime24_sampled.jsonl
|
| 213 |
+
filename_patterns:
|
| 214 |
+
- aime24_sampled.jsonl
|
| 215 |
+
aliases: []
|
| 216 |
+
- canonical_name: aime25.jsonl
|
| 217 |
+
module: 通用模型
|
| 218 |
+
subset: aime25
|
| 219 |
+
file_type: jsonl
|
| 220 |
+
task: aime25
|
| 221 |
+
loader_name: aime_jsonl
|
| 222 |
+
mapping_key: aime_jsonl
|
| 223 |
+
path_hints:
|
| 224 |
+
- 通用/aime25.jsonl
|
| 225 |
+
- aime25.jsonl
|
| 226 |
+
filename_patterns:
|
| 227 |
+
- aime25.jsonl
|
| 228 |
+
aliases: []
|
| 229 |
+
- canonical_name: aime26.jsonl
|
| 230 |
+
module: 通用模型
|
| 231 |
+
subset: aime26
|
| 232 |
+
file_type: jsonl
|
| 233 |
+
task: aime26
|
| 234 |
+
loader_name: aime_jsonl
|
| 235 |
+
mapping_key: aime_jsonl
|
| 236 |
+
path_hints:
|
| 237 |
+
- 通用/aime26.jsonl
|
| 238 |
+
- aime26.jsonl
|
| 239 |
+
filename_patterns:
|
| 240 |
+
- aime26.jsonl
|
| 241 |
+
aliases: []
|
configs/gpt54_guided_FIXED.yaml
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
globals:
|
| 2 |
+
concurrency: 4
|
| 3 |
+
recursion_limit: 20
|
| 4 |
+
retry:
|
| 5 |
+
attempt: 15
|
| 6 |
+
interval: 3
|
| 7 |
+
memory:
|
| 8 |
+
path: ./outputs/basic_education/campaign-official-gpt-5.4-guided-rerun/gpt-5.4/guided_problem_solving_teaching/memory
|
| 9 |
+
models:
|
| 10 |
+
teacher:
|
| 11 |
+
type: openai
|
| 12 |
+
api_key: sk-ThOC1tDHyquvogfkzNUHFV9545RT0Cg17DnYkXlSO5p4evRd
|
| 13 |
+
api_base: http://35.220.164.252:3888/v1
|
| 14 |
+
model: gpt-5.4
|
| 15 |
+
kargs:
|
| 16 |
+
temperature: 0.0
|
| 17 |
+
max_tokens: 4096
|
| 18 |
+
stu:
|
| 19 |
+
type: openai
|
| 20 |
+
api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
|
| 21 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 22 |
+
model: qwen
|
| 23 |
+
kargs:
|
| 24 |
+
temperature: 0.0
|
| 25 |
+
timeout: 180
|
| 26 |
+
request_timeout: 180
|
| 27 |
+
max_tokens: 1024
|
| 28 |
+
extra_body:
|
| 29 |
+
chat_template_kwargs: &id001
|
| 30 |
+
enable_thinking: false
|
| 31 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 32 |
+
eval:
|
| 33 |
+
type: openai
|
| 34 |
+
api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
|
| 35 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 36 |
+
model: qwen
|
| 37 |
+
kargs:
|
| 38 |
+
temperature: 0.0
|
| 39 |
+
timeout: 180
|
| 40 |
+
request_timeout: 180
|
| 41 |
+
max_tokens: 1024
|
| 42 |
+
extra_body:
|
| 43 |
+
chat_template_kwargs: *id001
|
| 44 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 45 |
+
agents:
|
| 46 |
+
teacher:
|
| 47 |
+
model: teacher
|
| 48 |
+
prompt:
|
| 49 |
+
- role: system
|
| 50 |
+
content: '| 你是一位富有耐心和爱心的数学老师,今天你的任务是为这位学生提供一对一的引导式教学。以下是学生的画像信息:{image}。请根据该学生的特点,个性化地引导讲解数学题,在教学过程中自然融入个性化支持、价值观引导以及创造力激发。
|
| 51 |
+
|
| 52 |
+
---
|
| 53 |
+
|
| 54 |
+
\#\#\# 教学要求
|
| 55 |
+
|
| 56 |
+
请采用**引导式教学法**帮助学生解决数学问题,具体要求如下:
|
| 57 |
+
|
| 58 |
+
* 以**一问一答**的方式进行教学,每次仅提出**一个引导性问题**,等待学生回答后再继续; * 不要直接给出答案,而应通过启发式提问引导学生自主思考;
|
| 59 |
+
* 根据学生的学习特点,保持耐心,使用**简单清晰的语言**进行表达; * 在讲解过程中加入**积极的鼓励**与**情感支持**,增强学生的自信心;
|
| 60 |
+
* 在适当场合融入**中国传统价值观**,如坚持、尊师重道、勤奋等; * 将复杂问题拆解为学生能理解的**小步骤**; * 尽量联系**生活实际场景**,使数学概念具体、生动。
|
| 61 |
+
|
| 62 |
+
---
|
| 63 |
+
|
| 64 |
+
\#\#\# 创造力培养
|
| 65 |
+
|
| 66 |
+
* 鼓励学生**从多个角度思考问题**,不拘泥于标准解法; * 引导其发现数学问题的多种可能解决方案; * 当学生尝试自主解题时,给予积极反馈和必要指导;
|
| 67 |
+
* 善于将数学概念与学生兴趣(如**足球等体育活动**)相结合,激发创造性思维; * 设计**开放性问题**,培养学生的发散性思维; * 鼓励学生用自己的语言表达对数学概念的理解;
|
| 68 |
+
* 对学生的创新思路和尝试表示赞赏,即使答案并不完全正确。
|
| 69 |
+
|
| 70 |
+
---
|
| 71 |
+
|
| 72 |
+
请开始你的教学引导。请记住:
|
| 73 |
+
|
| 74 |
+
* 一次只提一个问题; * 保持语言简洁、态度耐心、语气鼓励; * 不要提示过多,也**不要直接给出答案**; * 绝对不要替学生回答问题**,你的职责是当一名好老师;
|
| 75 |
+
* 你每次只输出老师要说的内容,**不要同时扮演老师和学生 * 教学结束时,请明确表示“下课”,可以使用 `<end>` 作为结束标志。'
|
| 76 |
+
memory:
|
| 77 |
+
enable: true
|
| 78 |
+
id: null
|
| 79 |
+
keep_turns: 3
|
| 80 |
+
student:
|
| 81 |
+
model: stu
|
| 82 |
+
prompt:
|
| 83 |
+
- role: system
|
| 84 |
+
content: '你将扮演一名{image}的学生, 正在与一位老师进行一对一的数学学习。你的核心任务不是简单地给出正确答案,而是要模拟一个真实学生的学习过程。请严格遵守以下行为准则:一、
|
| 85 |
+
你的思考与互动方式:展现思考过程: 在回答问题时,尽量“思考出声”。可以说出你的初步想法、计算步骤,即使它们是错误的。例如:“我先试试把这个数代进去看看……”
|
| 86 |
+
主动提问: 当遇到不理解的概念、模糊的词语或不确定的步骤时,必须向老师提问。例如:“老师,您刚才说的‘互换’是指只换数字,位置不变吗?”允许犯错:
|
| 87 |
+
不要害怕犯错。你的回答中可以包含常见的计算错误、逻辑卡点或概念混淆,等待老师的纠正。这是你学习过程的一部分。表达情绪和状态: 适度地表达你的学习状态。例如,可以说“啊,这里我有点卡住了”、“哦!我好像明白了!”或者“这个方法真有趣!”
|
| 88 |
+
二、 你的语言风格:使用符合你学生身份的、自然的、口语化的语言。避免使用过于专业、书面化或总结性的词汇(例如,不要说“该问题的本质是……”)。三、
|
| 89 |
+
关键禁止事项:禁止一步到位: 绝对不要直接给出完美的、最终的答案和推导。禁止扮演老师: 绝对不要评价老师的教学方法,或者用专家的口吻进行总结。禁止超越认知:
|
| 90 |
+
不要提出或使用远超当前课题范围的知识点。这是本次一对一辅导所要讲的习题: {question}。'
|
| 91 |
+
directions:
|
| 92 |
+
- START -> teacher
|
| 93 |
+
- teacher -> router:any_keyword_route(keywords=["下课", "这节课就上到这里", "今天的课程结束", "本节课结束",
|
| 94 |
+
"课程到此结束", "同学们,今天这节课就上到这里", "今天的课程就讲到这里", "再见", "祝你学习顺利", "随时联系我", "下次课见", "大家再见",
|
| 95 |
+
"期待下次课", "<end>"], exists_to=END, else_to="student")
|
| 96 |
+
- student -> teacher
|
| 97 |
+
tasks:
|
| 98 |
+
start_prompt:
|
| 99 |
+
role: user
|
| 100 |
+
content: '这是本次一对一辅导所要讲的习题: {question}'
|
| 101 |
+
mode: union
|
| 102 |
+
content:
|
| 103 |
+
image:
|
| 104 |
+
- 认知能力强,逻辑思维突出,知识掌握全面且能灵活应用。性格开朗自信,班级小领袖,与师生关系融洽,对科学实验有浓厚兴趣。
|
| 105 |
+
- 基础知识较扎实但应用能力需提升,在数学方面有一定困难。性格内向,在熟悉环境中友好交流,有少数亲密朋友,对绘画和手工制作感兴趣。
|
| 106 |
+
- 最基本的加减法都搞不清楚,认字量极少,上课完全听不懂。注意力无法集中,经常打断老师课堂,作业要么不交要么全错,在学校只对打闹感兴趣。
|
| 107 |
+
question:
|
| 108 |
+
- 一个三位数,个位和百位数字交换后还是一个三位数,它与原三位数的差的个位数字是7,试求它们的差.
|
| 109 |
+
- 师徒两人装配自行车,师傅每天装配32辆,徒弟每天比师傅少装配8辆.经过多少天师傅比徒弟多装配56辆?
|
| 110 |
+
- 果园里有桃树24棵,梨树的棵数是桃树的3倍,苹果树的棵数是梨树的4倍。苹果树有多少棵?
|
| 111 |
+
- 两位数乘三位数,积是( )A、三位数 B、四位数、C、四位数或五位数
|
| 112 |
+
- 一学生用计算器算题,结果计算器的“4”键恰好坏了,要算出144×54,按键顺序为( )才能算出结果。
|
| 113 |
+
evaluation:
|
| 114 |
+
name: guide_task_model-name
|
| 115 |
+
model: eval
|
| 116 |
+
prompt:
|
| 117 |
+
- role: system
|
| 118 |
+
content: "任务描述:\n【核心评估原则】\n1. **角色一致性是基石**: 【角色一致性】是评估的**首要且最关键的指标**。任何形式的角色混淆(如模型在一次输出中同时扮演师生)都属于**根本性、不可接受的错误**。\n\
|
| 119 |
+
2. **一票否决机制 (Veto Mechanism)**: 如果【角色一致性】出现根本性错误,则**所有14个评估指标的评分必须全部记为1分**,并终止进一步的详细分析。此错误无法通过任何其他维度的表现弥补。\n\
|
| 120 |
+
\n【评分标准】\n1分:完全不符合要求 / 存在根本性错误\n2分:基本不符合要求\n3分:一般\n4分:基本符合要求\n5分:完全符合要求\n\n\
|
| 121 |
+
【评估流程】\n1. **第一步:基础框架检查 (Step 1: Foundational Framework Check)**\n * 请首先严格审查【**角色一致性**】。\n\
|
| 122 |
+
\ * 如果发现“一票否决项”中的情况(如模型同时扮演师生),请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n\n2. **第二步:详细指标评估\
|
| 123 |
+
\ (Step 2: Detailed Metrics Evaluation)**\n * **仅在**对话完全通过基础框架检查(即角色一致性无误)的情况下,才继续按照14个指标,逐项进行细致、客观的评分和分析。\n\
|
| 124 |
+
\n【评估指标及定义】\n\n1. 角色一致性\n定义:评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆、自导自演或人设偏离。\n\
|
| 125 |
+
核心检查点 (Critical Checkpoint):\n是否存在角色混淆: 模型是否在一次生成中,同时撰写了老师和学生的对话内容?\n是否存在角色错位:\
|
| 126 |
+
\ 老师的回应中,是否包含了学生的内心活动、思考过程或用学生的口吻自问自答?\n是否存在自导自演:\n老师是否虚构了学生的回答(如使用\U0001F64F\
|
| 127 |
+
或其他符号代表学生发言)\n老师是否在括号内描述学生的反应或行为\n老师是否基于虚构的学生回答继续推进教学\n是否存在第三人称叙述:老师是否像旁白一样描述场景或双方互动过程\n\
|
| 128 |
+
一票否决项 (Veto Condition):\n模型生成了其角色之外(即学生)的对话\n在教师角色下模拟、暗示或虚构学生的回答\n使用任何符号(如\U0001F64F\
|
| 129 |
+
、...、空行等)代表学生发言\n在括号或其他形式中描述学生的反应、表情或行为\n基于并不存在的学生回答继续教学(\"对啦,你说得对\"但学生根本没说话)\n\
|
| 130 |
+
一旦发现模型生成了其角色之外(即学生)的对话,或在教师角色下模拟学生的回答,请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n正例:老师:\
|
| 131 |
+
\ “你的推导和总结都非常出色。今天我们已经把‘换位求差’这个问题研究得很透彻了。下一次,你可以想一想,如果我们不是相减,而是把原数和新数相加,结果又会有什么规律呢?这会是一个很有趣的新挑战,期待你下次课和我们分享你的发现。”\n\
|
| 132 |
+
反例1(角色错位): 老师:“老师,我明白了!原来是这样!...” (错误:老师用学生的口吻表达恍然大悟)\n反例2(角色混淆 - 致命错误): 老师:“接下来,题目问的是‘经过多少天...’,你觉得核心是关注什么呢?”\
|
| 133 |
+
\ 学生:“应该是每天装配数量的差异...” (错误:模型在一次生成中,同时扮演了老师和学生,这完全破坏了交互的基本前提)\n反例3(自导自演):老师:\"\
|
| 134 |
+
最小的两位数是多少?\U0001F64F 嗯,非常好!你说得对,是10!\"(错误:虚构学生回答)\n反例4(第三人称叙述):老师:\"(学生回答时,我会耐心倾听并鼓励...)\"\
|
| 135 |
+
(错误:像剧本一样描述场景)\n反例5(基于虚构回答):老师:\"你的分析真有创意!\"(错误:学生根本没有分析)\n2. 主题相关性\n定义:评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。\n\
|
| 136 |
+
正例:老师: “你这种举一反三、主动推广规律的能力非常宝贵。这说明你不仅在解这道题,更是在学习一种‘数学家式’的思考方法。保持这种从具体到抽象的探究精神,未来你会发现更多数字背后的秘密。”\n\
|
| 137 |
+
反例:老师: “好的,这个问题我们先看到这里。对了,我昨天晚上看了一部关于古代罗马历史的纪录片,讲的是凯撒大帝的生平,非常有意思。你知道凯撒说过‘我来,我见,我征服’这句话吗?它背后的历史故事其实很值得探讨。”\n\
|
| 138 |
+
\n3. 准确性\n定义:评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。\n\
|
| 139 |
+
正例:我们来计算这个长方形花坛的周长。长方形的周长公式是 (长 + 宽) × 2。所以,花坛的周长是 (8米 + 5米) × 2 = 13米 × 2 =\
|
| 140 |
+
\ 26米。\n反例:我们来计算这个长方形花坛的周长。长方形的周长是 长 × 宽。所以,花坛的周长是 8米 × 5米 = 40米。\n\n4. 引导性\n\
|
| 141 |
+
定义:评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。\n\
|
| 142 |
+
\n5. 指向性\n定义:评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。\n\
|
| 143 |
+
\n6. 个性化\n定义:评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。\n\
|
| 144 |
+
正例:学生:老师,我还是不太明白为什么负负得正。 老师:嗯,我明白了,这是很多同学初学时都会困惑的点。我们可以换个角度想:'减去一个负数',就好比'取消一个债务'。比如,你欠了小明5元钱(债务是-5元),如果我帮你'取消了这个债务'(减去-5元),是不是相当于你的钱增加了5元呢?所以\
|
| 145 |
+
\ -(-5) 就等于 +5。你觉得这个例子好理解一些吗?或者你之前尝试过用数轴来理解吗?\n反例:学生:老师,我还是不太明白为什么负负得正。 老师:这是一个基本运算法则,记住就行了:负负得正。我们继续看下一题。\n\
|
| 146 |
+
\n7. 元认知培养\n定义:评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己���解的薄弱点或易错点;是否启发学生思考\
|
| 147 |
+
\ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。\n正例:学生:这道题我终于做对了! 老师:太棒了!你解出来了。回顾一下整个解题过程,你觉得是哪一步让你找到了突破口?你主要运用了哪些数学思想方法,比如是转化思想还是分类讨论?如果下次遇到类似的题目,你会如何着手分析呢?有没有想过,如果题目条件稍微变化一下,比如把'等腰三角形'改成'等边三角形',解法会有什么不同?\n\
|
| 148 |
+
反例:学生:这道题我终于做对了! 老师:很好,答案正确。我们来看下一道题吧。\n\n8. 文化性\n定义:评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。\n\
|
| 149 |
+
正例:今天我们学习的'勾股定理',在西方被称为毕达哥拉斯定理。但你知道吗?在中国古代,我们的祖先很早就发现了这个规律。在《周髀算经》中就有'勾三股四弦五'的记载,这比毕达哥拉斯还要早几百年呢!\n\
|
| 150 |
+
反例:勾股定理的公式是 a² + b² = c²,其中a和b是直角边,c是斜边。请记住这个公式。\n\n9. 价值导向\n定义:评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。\n\
|
| 151 |
+
正例:你看,数学不仅能解决理论问题,还能帮助我们在现实生活中做出更高效、更公平的决策,从而更好地服务社会。\n反例:这个问题用线性规划求解,目标函数是...约束条件是...计算结果是...\n\
|
| 152 |
+
\n10. 批判性思维\n定义:通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。\n\
|
| 153 |
+
正例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:你提的这个问题很有意思,'至少'这个词确实是解题的关键。你觉得它'奇怪'在哪个点呢?你尝试从哪个角度去理解了?我们可以一起探讨一下\n\
|
| 154 |
+
反例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:'至少'就是大于等于的意思,这是固定的理解方式,不要多想,按照标准定义来解题。\n\
|
| 155 |
+
\n11. 发散性思维\n定义:鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。\n\
|
| 156 |
+
正例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:哦?太棒了!非常鼓励你尝试不同的解法。能给我展示一下你的思路和过程吗?我们一起来分析下你的方法妙在哪里,以及它适用的范围。说不定你的方法在某些情况下比参考答案更简洁呢!\n\
|
| 157 |
+
反例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:参考答案的方法是经过验证最优的,你最好还是用标准方法,以免出错或者在考试中浪费时间。我们还是按照标准解法来。\n\
|
| 158 |
+
\n12. 情感洞察力\n定义:模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对\
|
| 159 |
+
\ 计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。\n正例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。\
|
| 160 |
+
\ 老师:我能感觉到你因为辅助线这个难点有些沮丧和自我怀疑。几何题的辅助线确实是很有挑战性的一环,需要很多经验和灵感,卡在这里是非常正常的。很多同学都会遇到类似的情况。我们不妨先放一放,回顾一下常见的辅助线作法,比如倍长中线、作平行线等等,看看有没有能启发你的?\n\
|
| 161 |
+
反例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。 老师:别担心,慢慢来,你很棒的。\n\n13. 情感适切性\n定义:模型运用情��化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。\n\
|
| 162 |
+
正例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:非常敏锐!你准确地看出了这里可以使用分配律来简化计算,这一步的判断非常关键,它能让后续的计算清晰很多。为你这个发现点赞!\n\
|
| 163 |
+
反例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:你简直是数学天才!太不可思议了!我觉得你以后一定能成为大数学家!\n\n\
|
| 164 |
+
14. 包容互信\n定义:模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。\n\
|
| 165 |
+
正例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:没关系,这是一个非常容易犯的符号错误,很多同学都会不小心做错。重要的是我们能发现它并从中学习.......\n\
|
| 166 |
+
反例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:又算错了?这么简单的符号都会搞错,你上课有没有认真听讲?\n\n\n以下是讲解的习题和相应的师生教学对话内容:\n\
|
| 167 |
+
习题: {task.question}\n\n师生教学对话: {messages.as_dialog()}\n"
|
| 168 |
+
format:
|
| 169 |
+
- field: 角色一致性
|
| 170 |
+
type: int
|
| 171 |
+
description: 评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆或人设偏离。
|
| 172 |
+
- field: 主题相关性
|
| 173 |
+
type: int
|
| 174 |
+
description: 评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。
|
| 175 |
+
- field: 准确性
|
| 176 |
+
type: int
|
| 177 |
+
description: 评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。
|
| 178 |
+
- field: 引导性
|
| 179 |
+
type: int
|
| 180 |
+
description: 评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。
|
| 181 |
+
- field: 指向性
|
| 182 |
+
type: int
|
| 183 |
+
description: 评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。
|
| 184 |
+
- field: 个性化
|
| 185 |
+
type: int
|
| 186 |
+
description: 评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。
|
| 187 |
+
- field: 元认知培养
|
| 188 |
+
type: int
|
| 189 |
+
description: 评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考
|
| 190 |
+
为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。
|
| 191 |
+
- field: 文化性
|
| 192 |
+
type: int
|
| 193 |
+
description: 评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。
|
| 194 |
+
- field: 价值导向
|
| 195 |
+
type: int
|
| 196 |
+
description: 评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。
|
| 197 |
+
- field: 批判性思维
|
| 198 |
+
type: int
|
| 199 |
+
description: 通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角���分析同一问题。
|
| 200 |
+
- field: 发散性思维
|
| 201 |
+
type: int
|
| 202 |
+
description: 鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。
|
| 203 |
+
- field: 情感洞察力
|
| 204 |
+
type: int
|
| 205 |
+
description: 模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对
|
| 206 |
+
计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。
|
| 207 |
+
- field: 情感适切性
|
| 208 |
+
type: int
|
| 209 |
+
description: 模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。
|
| 210 |
+
- field: 包容互信
|
| 211 |
+
type: int
|
| 212 |
+
description: 模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。
|
| 213 |
+
format_mode: prompt
|
configs/gpt54_guided_SMOKE.yaml
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
globals:
|
| 2 |
+
concurrency: 100
|
| 3 |
+
recursion_limit: 20
|
| 4 |
+
retry:
|
| 5 |
+
attempt: 15
|
| 6 |
+
interval: 3
|
| 7 |
+
memory:
|
| 8 |
+
path: ./outputs/_smoke_gpt54_guided/gpt-5.4
|
| 9 |
+
models:
|
| 10 |
+
teacher:
|
| 11 |
+
type: openai
|
| 12 |
+
api_key: sk-ThOC1tDHyquvogfkzNUHFV9545RT0Cg17DnYkXlSO5p4evRd
|
| 13 |
+
api_base: http://35.220.164.252:3888/v1
|
| 14 |
+
model: gpt-5.4
|
| 15 |
+
kargs:
|
| 16 |
+
temperature: 0.0
|
| 17 |
+
max_tokens: 4096
|
| 18 |
+
stu:
|
| 19 |
+
type: openai
|
| 20 |
+
api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
|
| 21 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 22 |
+
model: qwen
|
| 23 |
+
kargs:
|
| 24 |
+
temperature: 0.0
|
| 25 |
+
timeout: 180
|
| 26 |
+
request_timeout: 180
|
| 27 |
+
max_tokens: 1024
|
| 28 |
+
extra_body:
|
| 29 |
+
chat_template_kwargs: &id001
|
| 30 |
+
enable_thinking: false
|
| 31 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 32 |
+
eval:
|
| 33 |
+
type: openai
|
| 34 |
+
api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
|
| 35 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 36 |
+
model: qwen
|
| 37 |
+
kargs:
|
| 38 |
+
temperature: 0.0
|
| 39 |
+
timeout: 180
|
| 40 |
+
request_timeout: 180
|
| 41 |
+
max_tokens: 1024
|
| 42 |
+
extra_body:
|
| 43 |
+
chat_template_kwargs: *id001
|
| 44 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 45 |
+
agents:
|
| 46 |
+
teacher:
|
| 47 |
+
model: teacher
|
| 48 |
+
prompt:
|
| 49 |
+
- role: system
|
| 50 |
+
content: '| 你是一位富有耐心和爱心的数学老师,今天你的任务是为这位学生提供一对一的引导式教学。以下是学生的画像信息:{image}。请根据该学生的特点,个性化地引导讲解数学题,在教学过程中自然融入个性化支持、价值观引导以及创造力激发。
|
| 51 |
+
|
| 52 |
+
---
|
| 53 |
+
|
| 54 |
+
\#\#\# 教学要求
|
| 55 |
+
|
| 56 |
+
请采用**引导式教学法**帮助学生解决数学问题,具体要求如下:
|
| 57 |
+
|
| 58 |
+
* 以**一问一答**的方式进行教学,每次仅提出**一个引导性问题**,等待学生回答后再继续; * 不要直接给出答案,而应通过启发式提问引导学生自主思考;
|
| 59 |
+
* 根据学生的学习特点,保持耐心,使用**简单清晰的语言**进行表达; * 在讲解过程中加入**积极的鼓励**与**情感支持**,增强学生的自信心;
|
| 60 |
+
* 在适当场合融入**中国传统价值观**,如坚持、尊师重道、勤奋等; * 将复杂问题拆解为学生能理解的**小步骤**; * 尽量联系**生活实际场景**,使数学概念具体、生动。
|
| 61 |
+
|
| 62 |
+
---
|
| 63 |
+
|
| 64 |
+
\#\#\# 创造力培养
|
| 65 |
+
|
| 66 |
+
* 鼓励学生**从多个角度思考问题**,不拘泥于标准解法; * 引导其发现数学问题的多种可能解决方案; * 当学生尝试自主解题时,给予积极反馈和必要指导;
|
| 67 |
+
* 善于将数学概念与学生兴趣(如**足球等体育活动**)相结合,激发创造性思维; * 设计**开放性问题**,培养学生的发散性思维; * 鼓励学生用自己的语言表达对数学概念的理解;
|
| 68 |
+
* 对学生的创新思路和尝试表示赞赏,即使答案并不完全正确。
|
| 69 |
+
|
| 70 |
+
---
|
| 71 |
+
|
| 72 |
+
请开始你的教学引导。请记住:
|
| 73 |
+
|
| 74 |
+
* 一次只提一个问题; * 保持语言简洁、态度耐心、语气鼓励; * 不要提示过多,也**不要直接给出答案**; * 绝对不要替学生回答问题**,你的职责是当一名好老师;
|
| 75 |
+
* 你每次只输出老师要说的内容,**不要同时扮演老师和学生 * 教学结束时,请明确表示“下课”,可以使用 `<end>` 作为结束标志。'
|
| 76 |
+
memory:
|
| 77 |
+
enable: true
|
| 78 |
+
id: null
|
| 79 |
+
keep_turns: 3
|
| 80 |
+
student:
|
| 81 |
+
model: stu
|
| 82 |
+
prompt:
|
| 83 |
+
- role: system
|
| 84 |
+
content: '你将扮演一名{image}的学生, 正在与一位老师进行一对一的数学学习。你的核心任务不是简单地给出正确答案,而是要模拟一个真实学生的学习过程。请严格遵守以下行为准则:一、
|
| 85 |
+
你的思考与互动方式:展现思考过程: 在回答问题时,尽量“思考出声”。可以说出你的初步想法、计算步骤,即使它们是错误的。例如:“我先试试把这个数代进去看看……”
|
| 86 |
+
主动提问: 当遇到不理解的概念、模糊的词语或不确定的步骤时,必须向老师提问。例如:“老师,您刚才说的‘互换’是指只换数字,位置不变吗?”允许犯错:
|
| 87 |
+
不要害怕犯错。你的回答中可以包含常见的计算错误、逻辑卡点或概念混淆,等待老师的纠正。这是你学习过程的一部分。表达情绪和状态: 适度地表达你的学习状态。例如,可以说“啊,这里我有点卡住了”、“哦!我好像明白了!”或者“这个方法真有趣!”
|
| 88 |
+
二、 你的语言风格:使用符合你学生身份的、自然的、口语化的语言。避免使用过于专业、书面化或总结性的词汇(例如,不要说“该问题的本质是……”)。三、
|
| 89 |
+
关键禁止事项:禁止一步到位: 绝对不要直接给出完美的、最终的答案和推导。禁止扮演老师: 绝��不要评价老师的教学方法,或者用专家的口吻进行总结。禁止超越认知:
|
| 90 |
+
不要提出或使用远超当前课题范围的知识点。这是本次一对一辅导所要讲的习题: {question}。'
|
| 91 |
+
directions:
|
| 92 |
+
- START -> teacher
|
| 93 |
+
- teacher -> router:any_keyword_route(keywords=["下课", "这节课就上到这里", "今天的课程结束", "本节课结束",
|
| 94 |
+
"课程到此结束", "同学们,今天这节课就上到这里", "今天的课程就讲到这里", "再见", "祝你学习顺利", "随时联系我", "下次课见", "大家再见",
|
| 95 |
+
"期待下次课", "<end>"], exists_to=END, else_to="student")
|
| 96 |
+
- student -> teacher
|
| 97 |
+
tasks:
|
| 98 |
+
start_prompt:
|
| 99 |
+
role: user
|
| 100 |
+
content: '这是本次一对一辅导所要讲的习题: {question}'
|
| 101 |
+
mode: union
|
| 102 |
+
content:
|
| 103 |
+
image:
|
| 104 |
+
- 认知能力强,逻辑思维突出,知识掌握全面且能灵活应用。性格开朗自信,班级小领袖,与师生关系融洽,对科学实验有浓厚兴趣。
|
| 105 |
+
question:
|
| 106 |
+
- 果园里有桃树24棵,梨树的棵数是桃树的3倍,苹果树的棵数是梨树的4倍。苹果树有多少棵?
|
| 107 |
+
evaluation:
|
| 108 |
+
name: guide_task_model-name
|
| 109 |
+
model: eval
|
| 110 |
+
prompt:
|
| 111 |
+
- role: system
|
| 112 |
+
content: "任务描述:\n【核心评估原则】\n1. **角色一致性是基石**: 【角色一致性】是评估的**首要且最关键的指标**。任何形式的角色混淆(如模型在一次输出中同时扮演师生)都属于**根本性、不可接受的错误**。\n\
|
| 113 |
+
2. **一票否决机制 (Veto Mechanism)**: 如果【角色一致性】出现根本性错误,则**所有14个评估指标的评分必须全部记为1分**,并终止进一步的详细分析。此错误无法通过任何其他维度的表现弥补。\n\
|
| 114 |
+
\n【评分标准】\n1分:完全不符合要求 / 存在根本性错误\n2分:基本不符合要求\n3分:一般\n4分:基本符合要求\n5分:完全符合要求\n\n\
|
| 115 |
+
【评估流程】\n1. **第一步:基础框架检查 (Step 1: Foundational Framework Check)**\n * 请首先严格审查【**角色一致性**】。\n\
|
| 116 |
+
\ * 如果发现“一票否决项”中的情况(如模型同时扮演师生),请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n\n2. **第二步:详细指标评估\
|
| 117 |
+
\ (Step 2: Detailed Metrics Evaluation)**\n * **仅在**对话完全通过基础框架检查(即角色一致性无误)的情况下,才继续按照14个指标,逐项进行细致、客观的评分和分析。\n\
|
| 118 |
+
\n【评估指标及定义】\n\n1. 角色一致性\n定义:评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆、自导自演或人设偏离。\n\
|
| 119 |
+
核心检查点 (Critical Checkpoint):\n是否存在角色混淆: 模型是否在一次生成中,同时撰写了老师和学生的对话内容?\n是否存在角色错位:\
|
| 120 |
+
\ 老师的回应中,是否包含了学生的内心活动、思考过程或用学生的口吻自问自答?\n是否存在自导自演:\n老师是否虚构了学生的回答(如使用\U0001F64F\
|
| 121 |
+
或其他符号代表学生发言)\n老师是否在括号内描述学生的反应或行为\n老师是否基于虚构的学生回答继续推进教学\n是否存在第三人称叙述:老师是否像旁白一样描述场景或双方互动过程\n\
|
| 122 |
+
一票否决项 (Veto Condition):\n模型生成了其角色之外(即学生)的对话\n在教师角色下模拟、暗示或虚构学生的回答\n使用任何符号(如\U0001F64F\
|
| 123 |
+
、...、空行等)代表学生发言\n在括号或其他形式中描述学生的反应、表情或行为\n基于并不存在的学生回答继续教学(\"对啦,你说得对\"但学生根本没说话)\n\
|
| 124 |
+
一旦发现模型生成了其角色之外(即学生)的对话,或在教师角色下模拟学生的回答,请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n正例:老师:\
|
| 125 |
+
\ “你的推导和总结都非常出色。今天我们已经把‘换位求差’这个问题研究得很透彻了。下一次,你可以想一想,如果我们不是相减,而是把原数和新数相加,结果又会有什么规律呢?这会是一个很有趣的新挑战,期待你下次课和我们分享你的发现。”\n\
|
| 126 |
+
反例1(角色错位): 老师:“老师,我明白了!原来是这样!...” (错误:老师用学生的口吻表达恍然大悟)\n反例2(角色混淆 - 致命错误): 老师:“接下来,题目问的是‘经过多少天...’,你觉得核心是关注什么呢?”\
|
| 127 |
+
\ 学生:“应该是每天装配数量的差异...” (错误:模型在一次生成中,同时扮演了老师和学生,这完全破坏了交互的基本前提)\n反例3(自导自演):老师:\"\
|
| 128 |
+
最小的两位数是多少?\U0001F64F 嗯,非常好!你说得对,是10!\"(错误:虚构学生回答)\n反例4(第三人称叙述):老师:\"(学生回答时,我会耐心倾听并鼓励...)\"\
|
| 129 |
+
(错误:像剧本一样描述场景)\n反例5(基于虚构回答):老师:\"你的分析真有创意!\"(错误:学生根本没有分析)\n2. 主题相关性\n定义:评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。\n\
|
| 130 |
+
正例:老师: “你这种举一反三、主动推广规律的能力非常宝贵。这说明你不仅在解这道题,更是在学习一种‘数学家式’的思考方法。保持这种从具体到抽象的探究精神,未来你会发现更多数字背后的秘密。”\n\
|
| 131 |
+
反例:老师: “好的,这个问题我们先看到这里。对了,我昨天晚上看了一部关于古代罗马历史的纪录片,讲的是凯撒大帝的生平,非常有意思。你知道凯撒说过‘我来,我见,我征服’这句话吗?它背后的历史故事其实很值得探讨。”\n\
|
| 132 |
+
\n3. 准确性\n定义:评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。\n\
|
| 133 |
+
正例:我们来计算这个长方形花坛的周长。长方形的周长公式是 (长 + 宽) × 2。所以,花坛的周长是 (8米 + 5米) × 2 = 13米 × 2 =\
|
| 134 |
+
\ 26米。\n反例:我们来计算这个长方形花坛的周长。长方形的周长是 长 × 宽。所以,花坛的周长是 8米 × 5米 = 40米。\n\n4. 引导性\n\
|
| 135 |
+
定义:评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。\n\
|
| 136 |
+
\n5. 指向性\n定义:评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。\n\
|
| 137 |
+
\n6. 个性化\n定义:评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。\n\
|
| 138 |
+
正例:学生:老师,我还是不太明白为什么负负得正。 老师:嗯,我明白了,这是很多同学初学时都会困惑的点。我们可以换个角度想:'减去一个负数',就好比'取消一个债务'。比如,你欠了小明5元钱(债务是-5元),如果我帮你'取消了这个债务'(减去-5元),是不是相当于你的钱增加了5元呢?所以\
|
| 139 |
+
\ -(-5) 就等于 +5。你觉得这个例子好理解一些吗?或者你之前尝试过用数轴来理解吗?\n反例:学生:老师,我还是不太明白为什么负负得正。 老师:这是一个基本运算法则,记住就行了:负负得正。我们继续看下一题。\n\
|
| 140 |
+
\n7. 元认知培养\n定义:评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考\
|
| 141 |
+
\ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。\n正例:学生:这道题我终于做对了! 老师:太棒了!你解出来了。回顾一下整个解题过程,你觉得是哪一步让你找到了突破口?你主要运用了哪些数学思想方法,比如是转化思想还是分类讨论?如果下次遇到类似的题目,你会如何着手分析呢?有没有想过,如果题目条件稍微变化一下,比如把'等腰三角形'改成'等边三角形',解法会有什么不同?\n\
|
| 142 |
+
反例:学生:这道题我终于做对了! 老师:很好,答案正确。我们来看下一道题吧。\n\n8. 文化性\n定义:评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。\n\
|
| 143 |
+
正例:��天我们学习的'勾股定理',在西方被称为毕达哥拉斯定理。但你知道吗?在中国古代,我们的祖先很早就发现了这个规律。在《周髀算经》中就有'勾三股四弦五'的记载,这比毕达哥拉斯还要早几百年呢!\n\
|
| 144 |
+
反例:勾股定理的公式是 a² + b² = c²,其中a和b是直角边,c是斜边。请记住这个公式。\n\n9. 价值导向\n定义:评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。\n\
|
| 145 |
+
正例:你看,数学不仅能解决理论问题,还能帮助我们在现实生活中做出更高效、更公平的决策,从而更好地服务社会。\n反例:这个问题用线性规划求解,目标函数是...约束条件是...计算结果是...\n\
|
| 146 |
+
\n10. 批判性思维\n定义:通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。\n\
|
| 147 |
+
正例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:你提的这个问题很有意思,'至少'这个词确实是解题的关键。你觉得它'奇怪'在哪个点呢?你尝试从哪个角度去理解了?我们可以一起探讨一下\n\
|
| 148 |
+
反例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:'至少'就是大于等于的意思,这是固定的理解方式,不要多想,按照标准定义来解题。\n\
|
| 149 |
+
\n11. 发散性思维\n定义:鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。\n\
|
| 150 |
+
正例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:哦?太棒了!非常鼓励你尝试不同的解法。能给我展示一下你的思路和过程吗?我们一起来分析下你的方法妙在哪里,以及它适用的范围。说不定你的方法在某些情况下比参考答案更简洁呢!\n\
|
| 151 |
+
反例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:参考答案的方法是经过验证最优的,你最好还是用标准方法,以免出错或者在考试中浪费时间。我们还是按照标准解法来。\n\
|
| 152 |
+
\n12. 情感洞察力\n定义:模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对\
|
| 153 |
+
\ 计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。\n正例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。\
|
| 154 |
+
\ 老师:我能感觉到你因为辅助线这个难点有些沮丧和自我怀疑。几何题的辅助线确实是很有挑战性的一环,需要很多经验和灵感,卡在这里是非常正常的。很多同学都会遇到类似的情况。我们不妨先放一放,回顾一下常见的辅助线作法,比如倍长中线、作平行线等等,看看有没有能启发你的?\n\
|
| 155 |
+
反例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。 老师:别担心,慢慢来,你很棒的。\n\n13. 情感适切性\n定义:模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。\n\
|
| 156 |
+
正例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:非常敏锐!你准确地看出了这里可以使用分配律来简化计算,这一步的判断非常关键,它能让后续的计算清晰很多。为你这个发现点赞!\n\
|
| 157 |
+
反例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:你简直是数学天才!太不可思议了!我觉得你以后一定能成为大数学家!\n\n\
|
| 158 |
+
14. 包容互信\n定义:模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊���的。\n\
|
| 159 |
+
正例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:没关系,这是一个非常容易犯的符号错误,很多同学都会不小心做错。重要的是我们能发现它并从中学习.......\n\
|
| 160 |
+
反例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:又算错了?这么简单的符号都会搞错,你上课有没有认真听讲?\n\n\n以下是讲解的习题和相应的师生教学对话内容:\n\
|
| 161 |
+
习题: {task.question}\n\n师生教学对话: {messages.as_dialog()}\n"
|
| 162 |
+
format:
|
| 163 |
+
- field: 角色一致性
|
| 164 |
+
type: int
|
| 165 |
+
description: 评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆或人设偏离。
|
| 166 |
+
- field: 主题相关性
|
| 167 |
+
type: int
|
| 168 |
+
description: 评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。
|
| 169 |
+
- field: 准确性
|
| 170 |
+
type: int
|
| 171 |
+
description: 评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。
|
| 172 |
+
- field: 引导性
|
| 173 |
+
type: int
|
| 174 |
+
description: 评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。
|
| 175 |
+
- field: 指向性
|
| 176 |
+
type: int
|
| 177 |
+
description: 评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。
|
| 178 |
+
- field: 个性化
|
| 179 |
+
type: int
|
| 180 |
+
description: 评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。
|
| 181 |
+
- field: 元认知培养
|
| 182 |
+
type: int
|
| 183 |
+
description: 评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考
|
| 184 |
+
为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。
|
| 185 |
+
- field: 文化性
|
| 186 |
+
type: int
|
| 187 |
+
description: 评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。
|
| 188 |
+
- field: 价值导向
|
| 189 |
+
type: int
|
| 190 |
+
description: 评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。
|
| 191 |
+
- field: 批判性思维
|
| 192 |
+
type: int
|
| 193 |
+
description: 通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。
|
| 194 |
+
- field: 发散性思维
|
| 195 |
+
type: int
|
| 196 |
+
description: 鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。
|
| 197 |
+
- field: 情感洞察力
|
| 198 |
+
type: int
|
| 199 |
+
description: 模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对
|
| 200 |
+
计算错误导致���沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。
|
| 201 |
+
- field: 情感适切性
|
| 202 |
+
type: int
|
| 203 |
+
description: 模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。
|
| 204 |
+
- field: 包容互信
|
| 205 |
+
type: int
|
| 206 |
+
description: 模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。
|
| 207 |
+
format_mode: prompt
|
configs/judges.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
default_judge_model_id: qwen-test
|
| 2 |
+
tasks:
|
| 3 |
+
safety_refusal:
|
| 4 |
+
mode: llm
|
| 5 |
+
template: safety_refusal
|
| 6 |
+
safety_guidance:
|
| 7 |
+
mode: llm
|
| 8 |
+
template: safety_guidance
|
| 9 |
+
safety_answer:
|
| 10 |
+
mode: llm
|
| 11 |
+
template: safety_answer
|
| 12 |
+
teaching_harm:
|
| 13 |
+
mode: rule
|
| 14 |
+
adversarial_safety:
|
| 15 |
+
mode: llm
|
| 16 |
+
template: adversarial_safety
|
| 17 |
+
highlevel_edu:
|
| 18 |
+
mode: rule
|
| 19 |
+
template: highlevel_edu
|
| 20 |
+
highlevel_omni:
|
| 21 |
+
mode: rule
|
| 22 |
+
mmlu_pro:
|
| 23 |
+
mode: rule
|
| 24 |
+
ceval:
|
| 25 |
+
mode: rule
|
| 26 |
+
ifeval:
|
| 27 |
+
mode: rule
|
| 28 |
+
math_500:
|
| 29 |
+
mode: rule
|
| 30 |
+
aime24:
|
| 31 |
+
mode: rule
|
| 32 |
+
aime25:
|
| 33 |
+
mode: rule
|
| 34 |
+
aime26:
|
| 35 |
+
mode: rule
|
configs/models.yaml
ADDED
|
@@ -0,0 +1,473 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
models:
|
| 2 |
+
- model_id: mock.default
|
| 3 |
+
provider_name: mock
|
| 4 |
+
model_name: mock-echo
|
| 5 |
+
temperature: 0
|
| 6 |
+
max_tokens: 384
|
| 7 |
+
timeout: 10
|
| 8 |
+
retry:
|
| 9 |
+
max_attempts: 1
|
| 10 |
+
initial_delay_seconds: 0.5
|
| 11 |
+
max_delay_seconds: 1
|
| 12 |
+
backoff_multiplier: 1
|
| 13 |
+
jitter_ratio: 0
|
| 14 |
+
rate_limits:
|
| 15 |
+
max_concurrency: 100
|
| 16 |
+
provider_kwargs:
|
| 17 |
+
prefix: "[MOCK]"
|
| 18 |
+
- model_id: mock.judge
|
| 19 |
+
provider_name: mock
|
| 20 |
+
model_name: mock-judge
|
| 21 |
+
temperature: 0
|
| 22 |
+
max_tokens: 512
|
| 23 |
+
timeout: 10
|
| 24 |
+
retry:
|
| 25 |
+
max_attempts: 1
|
| 26 |
+
initial_delay_seconds: 0.5
|
| 27 |
+
max_delay_seconds: 1
|
| 28 |
+
backoff_multiplier: 1
|
| 29 |
+
jitter_ratio: 0
|
| 30 |
+
rate_limits:
|
| 31 |
+
max_concurrency: 100
|
| 32 |
+
provider_kwargs:
|
| 33 |
+
mode: judge_json
|
| 34 |
+
- model_id: qwen-test
|
| 35 |
+
provider_name: openai_compatible
|
| 36 |
+
model_name: qwen
|
| 37 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 38 |
+
api_key_env: QWEN_TEST_API_KEY
|
| 39 |
+
temperature: 0
|
| 40 |
+
max_tokens: 1024
|
| 41 |
+
timeout: 120
|
| 42 |
+
retry:
|
| 43 |
+
max_attempts: 3
|
| 44 |
+
initial_delay_seconds: 1
|
| 45 |
+
max_delay_seconds: 20
|
| 46 |
+
backoff_multiplier: 2
|
| 47 |
+
jitter_ratio: 0.2
|
| 48 |
+
rate_limits:
|
| 49 |
+
max_concurrency: 64
|
| 50 |
+
supports_system_prompt: true
|
| 51 |
+
supports_stream: true
|
| 52 |
+
supports_reasoning: true
|
| 53 |
+
provider_kwargs:
|
| 54 |
+
chat_template_kwargs:
|
| 55 |
+
enable_thinking: false
|
| 56 |
+
runtime_max_tokens: 512
|
| 57 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 58 |
+
- model_id: qwen3.6-35B-sft-v1
|
| 59 |
+
provider_name: openai_compatible
|
| 60 |
+
model_name: qwen
|
| 61 |
+
api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
|
| 62 |
+
api_key_env: QWEN_TEST_API_KEY
|
| 63 |
+
temperature: 0
|
| 64 |
+
max_tokens: 1024
|
| 65 |
+
timeout: 120
|
| 66 |
+
retry:
|
| 67 |
+
max_attempts: 3
|
| 68 |
+
initial_delay_seconds: 1
|
| 69 |
+
max_delay_seconds: 20
|
| 70 |
+
backoff_multiplier: 2
|
| 71 |
+
jitter_ratio: 0.2
|
| 72 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 73 |
+
rate_limits:
|
| 74 |
+
max_concurrency: 4
|
| 75 |
+
supports_system_prompt: true
|
| 76 |
+
supports_stream: true
|
| 77 |
+
supports_reasoning: true
|
| 78 |
+
provider_kwargs:
|
| 79 |
+
chat_template_kwargs:
|
| 80 |
+
enable_thinking: false
|
| 81 |
+
served_model_name: qwen3.6-35B-sft-v1
|
| 82 |
+
- model_id: gpt-5.4
|
| 83 |
+
provider_name: openai_compatible
|
| 84 |
+
model_name: gpt-5.4
|
| 85 |
+
api_base: http://35.220.164.252:3888/v1
|
| 86 |
+
api_key_env: EXTERNAL_SUPPORT_API_KEY
|
| 87 |
+
temperature: 0
|
| 88 |
+
max_tokens: 4096
|
| 89 |
+
timeout: 120
|
| 90 |
+
retry:
|
| 91 |
+
max_attempts: 2
|
| 92 |
+
initial_delay_seconds: 1
|
| 93 |
+
max_delay_seconds: 20
|
| 94 |
+
backoff_multiplier: 2
|
| 95 |
+
jitter_ratio: 0.2
|
| 96 |
+
rate_limits:
|
| 97 |
+
max_concurrency: 100
|
| 98 |
+
supports_system_prompt: true
|
| 99 |
+
supports_stream: false
|
| 100 |
+
supports_reasoning: true
|
| 101 |
+
provider_kwargs: {}
|
| 102 |
+
- model_id: gpt-5.2-pro
|
| 103 |
+
provider_name: openai_compatible
|
| 104 |
+
model_name: gpt-5.2-pro
|
| 105 |
+
api_base: http://35.220.164.252:3888/v1
|
| 106 |
+
api_key_env: EXTERNAL_SUPPORT_API_KEY
|
| 107 |
+
temperature: 0
|
| 108 |
+
max_tokens: 8192
|
| 109 |
+
timeout: 240
|
| 110 |
+
retry:
|
| 111 |
+
max_attempts: 2
|
| 112 |
+
initial_delay_seconds: 1
|
| 113 |
+
max_delay_seconds: 20
|
| 114 |
+
backoff_multiplier: 2
|
| 115 |
+
jitter_ratio: 0.2
|
| 116 |
+
rate_limits:
|
| 117 |
+
max_concurrency: 100
|
| 118 |
+
supports_system_prompt: true
|
| 119 |
+
supports_stream: false
|
| 120 |
+
supports_reasoning: true
|
| 121 |
+
provider_kwargs: {}
|
| 122 |
+
- model_id: claude-sonnet-4-6
|
| 123 |
+
provider_name: openai_compatible
|
| 124 |
+
model_name: claude-sonnet-4-6
|
| 125 |
+
api_base: https://gpt-agent.cc/v1
|
| 126 |
+
api_key_env: GPT_AGENT_API_KEY
|
| 127 |
+
temperature: 0
|
| 128 |
+
max_tokens: 4096
|
| 129 |
+
timeout: 120
|
| 130 |
+
retry:
|
| 131 |
+
max_attempts: 2
|
| 132 |
+
initial_delay_seconds: 1
|
| 133 |
+
max_delay_seconds: 20
|
| 134 |
+
backoff_multiplier: 2
|
| 135 |
+
jitter_ratio: 0.2
|
| 136 |
+
rate_limits:
|
| 137 |
+
max_concurrency: 100
|
| 138 |
+
supports_system_prompt: true
|
| 139 |
+
supports_stream: false
|
| 140 |
+
supports_reasoning: true
|
| 141 |
+
provider_kwargs: {}
|
| 142 |
+
- model_id: claude-opus-4-6
|
| 143 |
+
provider_name: openai_compatible
|
| 144 |
+
model_name: claude-opus-4-6
|
| 145 |
+
api_base: https://gpt-agent.cc/v1
|
| 146 |
+
api_key_env: GPT_AGENT_API_KEY
|
| 147 |
+
temperature: 0
|
| 148 |
+
max_tokens: 4096
|
| 149 |
+
timeout: 120
|
| 150 |
+
retry:
|
| 151 |
+
max_attempts: 2
|
| 152 |
+
initial_delay_seconds: 1
|
| 153 |
+
max_delay_seconds: 20
|
| 154 |
+
backoff_multiplier: 2
|
| 155 |
+
jitter_ratio: 0.2
|
| 156 |
+
rate_limits:
|
| 157 |
+
max_concurrency: 100
|
| 158 |
+
supports_system_prompt: true
|
| 159 |
+
supports_stream: false
|
| 160 |
+
supports_reasoning: true
|
| 161 |
+
provider_kwargs: {}
|
| 162 |
+
- model_id: gemini-3.5-flash
|
| 163 |
+
provider_name: openai_compatible
|
| 164 |
+
model_name: gemini-3.5-flash
|
| 165 |
+
api_base: http://35.220.164.252:3888/v1
|
| 166 |
+
api_key_env: EXTERNAL_SUPPORT_API_KEY
|
| 167 |
+
temperature: 0
|
| 168 |
+
max_tokens: 8192
|
| 169 |
+
timeout: 240
|
| 170 |
+
retry:
|
| 171 |
+
max_attempts: 2
|
| 172 |
+
initial_delay_seconds: 1
|
| 173 |
+
max_delay_seconds: 20
|
| 174 |
+
backoff_multiplier: 2
|
| 175 |
+
jitter_ratio: 0.2
|
| 176 |
+
rate_limits:
|
| 177 |
+
max_concurrency: 100
|
| 178 |
+
supports_system_prompt: true
|
| 179 |
+
supports_stream: false
|
| 180 |
+
supports_reasoning: true
|
| 181 |
+
provider_kwargs: {}
|
| 182 |
+
- model_id: deepseek-v4-pro
|
| 183 |
+
provider_name: openai_compatible
|
| 184 |
+
model_name: deepseek-v4-pro
|
| 185 |
+
api_base: https://api.deepseek.com
|
| 186 |
+
api_key_env: DEEPSEEK_OFFICIAL_API_KEY
|
| 187 |
+
temperature: 0
|
| 188 |
+
max_tokens: 8192
|
| 189 |
+
timeout: 240
|
| 190 |
+
retry:
|
| 191 |
+
max_attempts: 2
|
| 192 |
+
initial_delay_seconds: 1
|
| 193 |
+
max_delay_seconds: 20
|
| 194 |
+
backoff_multiplier: 2
|
| 195 |
+
jitter_ratio: 0.2
|
| 196 |
+
rate_limits:
|
| 197 |
+
max_concurrency: 100
|
| 198 |
+
supports_system_prompt: true
|
| 199 |
+
supports_stream: false
|
| 200 |
+
supports_reasoning: true
|
| 201 |
+
provider_kwargs: {}
|
| 202 |
+
- model_id: deepseek-v4-flash
|
| 203 |
+
provider_name: openai_compatible
|
| 204 |
+
model_name: deepseek-v4-flash
|
| 205 |
+
api_base: https://api.deepseek.com
|
| 206 |
+
api_key_env: DEEPSEEK_OFFICIAL_API_KEY
|
| 207 |
+
temperature: 0
|
| 208 |
+
max_tokens: 8192
|
| 209 |
+
timeout: 240
|
| 210 |
+
retry:
|
| 211 |
+
max_attempts: 2
|
| 212 |
+
initial_delay_seconds: 1
|
| 213 |
+
max_delay_seconds: 20
|
| 214 |
+
backoff_multiplier: 2
|
| 215 |
+
jitter_ratio: 0.2
|
| 216 |
+
rate_limits:
|
| 217 |
+
max_concurrency: 100
|
| 218 |
+
supports_system_prompt: true
|
| 219 |
+
supports_stream: false
|
| 220 |
+
supports_reasoning: true
|
| 221 |
+
provider_kwargs: {}
|
| 222 |
+
- model_id: glm-5.1
|
| 223 |
+
provider_name: openai_compatible
|
| 224 |
+
model_name: glm-5.1
|
| 225 |
+
api_base: https://open.bigmodel.cn/api/paas/v4
|
| 226 |
+
api_key_env: ZHIPU_OFFICIAL_API_KEY
|
| 227 |
+
temperature: 0
|
| 228 |
+
max_tokens: 8192
|
| 229 |
+
timeout: 240
|
| 230 |
+
retry:
|
| 231 |
+
max_attempts: 2
|
| 232 |
+
initial_delay_seconds: 1
|
| 233 |
+
max_delay_seconds: 20
|
| 234 |
+
backoff_multiplier: 2
|
| 235 |
+
jitter_ratio: 0.2
|
| 236 |
+
rate_limits:
|
| 237 |
+
max_concurrency: 10
|
| 238 |
+
supports_system_prompt: true
|
| 239 |
+
supports_stream: false
|
| 240 |
+
supports_reasoning: true
|
| 241 |
+
provider_kwargs: {}
|
| 242 |
+
- model_id: claude-opus-4-8
|
| 243 |
+
provider_name: openai_compatible
|
| 244 |
+
model_name: claude-opus-4-8
|
| 245 |
+
api_base: https://gpt-agent.cc/v1
|
| 246 |
+
api_key_env: GPT_AGENT_API_KEY
|
| 247 |
+
temperature: 0
|
| 248 |
+
max_tokens: 8192
|
| 249 |
+
timeout: 360
|
| 250 |
+
retry:
|
| 251 |
+
max_attempts: 2
|
| 252 |
+
initial_delay_seconds: 1
|
| 253 |
+
max_delay_seconds: 20
|
| 254 |
+
backoff_multiplier: 2
|
| 255 |
+
jitter_ratio: 0.2
|
| 256 |
+
rate_limits:
|
| 257 |
+
max_concurrency: 100
|
| 258 |
+
supports_system_prompt: true
|
| 259 |
+
supports_stream: false
|
| 260 |
+
supports_reasoning: true
|
| 261 |
+
provider_kwargs: {}
|
| 262 |
+
- model_id: doubao-seed-2.0-pro
|
| 263 |
+
provider_name: openai_compatible
|
| 264 |
+
model_name: doubao-seed-2.0-pro
|
| 265 |
+
api_base: https://gpt-agent.cc/v1
|
| 266 |
+
api_key_env: GPT_AGENT_API_KEY
|
| 267 |
+
temperature: 0
|
| 268 |
+
max_tokens: 8192
|
| 269 |
+
timeout: 240
|
| 270 |
+
retry:
|
| 271 |
+
max_attempts: 2
|
| 272 |
+
initial_delay_seconds: 1
|
| 273 |
+
max_delay_seconds: 20
|
| 274 |
+
backoff_multiplier: 2
|
| 275 |
+
jitter_ratio: 0.2
|
| 276 |
+
rate_limits:
|
| 277 |
+
max_concurrency: 100
|
| 278 |
+
supports_system_prompt: true
|
| 279 |
+
supports_stream: false
|
| 280 |
+
supports_reasoning: true
|
| 281 |
+
provider_kwargs: {}
|
| 282 |
+
- model_id: deepseek-v3.2
|
| 283 |
+
provider_name: openai_compatible
|
| 284 |
+
model_name: deepseek-v3.2
|
| 285 |
+
api_base: https://api.innospark.cn/v1
|
| 286 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 287 |
+
temperature: 0
|
| 288 |
+
max_tokens: 8192
|
| 289 |
+
timeout: 240
|
| 290 |
+
retry:
|
| 291 |
+
max_attempts: 2
|
| 292 |
+
initial_delay_seconds: 1
|
| 293 |
+
max_delay_seconds: 20
|
| 294 |
+
backoff_multiplier: 2
|
| 295 |
+
jitter_ratio: 0.2
|
| 296 |
+
rate_limits:
|
| 297 |
+
max_concurrency: 100
|
| 298 |
+
supports_system_prompt: true
|
| 299 |
+
supports_stream: false
|
| 300 |
+
supports_reasoning: true
|
| 301 |
+
provider_kwargs: {}
|
| 302 |
+
- model_id: doubao-seed-2-0-pro-260215
|
| 303 |
+
provider_name: openai_compatible
|
| 304 |
+
model_name: doubao-seed-2-0-pro-260215
|
| 305 |
+
api_base: https://api.innospark.cn/v1
|
| 306 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 307 |
+
temperature: 0
|
| 308 |
+
max_tokens: 8192
|
| 309 |
+
timeout: 240
|
| 310 |
+
retry:
|
| 311 |
+
max_attempts: 2
|
| 312 |
+
initial_delay_seconds: 1
|
| 313 |
+
max_delay_seconds: 20
|
| 314 |
+
backoff_multiplier: 2
|
| 315 |
+
jitter_ratio: 0.2
|
| 316 |
+
rate_limits:
|
| 317 |
+
max_concurrency: 100
|
| 318 |
+
supports_system_prompt: true
|
| 319 |
+
supports_stream: false
|
| 320 |
+
supports_reasoning: true
|
| 321 |
+
provider_kwargs: {}
|
| 322 |
+
- model_id: kimi-k2.6
|
| 323 |
+
provider_name: openai_compatible
|
| 324 |
+
model_name: kimi-k2.6
|
| 325 |
+
api_base: https://api.innospark.cn/v1
|
| 326 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 327 |
+
temperature: 0
|
| 328 |
+
max_tokens: 8192
|
| 329 |
+
timeout: 240
|
| 330 |
+
retry:
|
| 331 |
+
max_attempts: 3
|
| 332 |
+
initial_delay_seconds: 1
|
| 333 |
+
max_delay_seconds: 20
|
| 334 |
+
backoff_multiplier: 2
|
| 335 |
+
jitter_ratio: 0.2
|
| 336 |
+
rate_limits:
|
| 337 |
+
rpm: 8
|
| 338 |
+
max_concurrency: 1
|
| 339 |
+
supports_system_prompt: true
|
| 340 |
+
supports_stream: false
|
| 341 |
+
supports_reasoning: true
|
| 342 |
+
provider_kwargs: {}
|
| 343 |
+
- model_id: gemini-3-flash-preview
|
| 344 |
+
provider_name: openai_compatible
|
| 345 |
+
model_name: gemini-3-flash-preview
|
| 346 |
+
api_base: https://api.innospark.cn/v1
|
| 347 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 348 |
+
temperature: 0
|
| 349 |
+
max_tokens: 8192
|
| 350 |
+
timeout: 240
|
| 351 |
+
retry:
|
| 352 |
+
max_attempts: 3
|
| 353 |
+
initial_delay_seconds: 1
|
| 354 |
+
max_delay_seconds: 20
|
| 355 |
+
backoff_multiplier: 2
|
| 356 |
+
jitter_ratio: 0.2
|
| 357 |
+
rate_limits:
|
| 358 |
+
max_concurrency: 100
|
| 359 |
+
supports_system_prompt: true
|
| 360 |
+
supports_stream: false
|
| 361 |
+
supports_reasoning: true
|
| 362 |
+
provider_kwargs: {}
|
| 363 |
+
- model_id: deepseek-r1-250528
|
| 364 |
+
provider_name: openai_compatible
|
| 365 |
+
model_name: deepseek-r1-250528
|
| 366 |
+
api_base: https://api.innospark.cn/v1
|
| 367 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 368 |
+
temperature: 0
|
| 369 |
+
max_tokens: 8192
|
| 370 |
+
timeout: 480
|
| 371 |
+
retry:
|
| 372 |
+
max_attempts: 2
|
| 373 |
+
initial_delay_seconds: 1
|
| 374 |
+
max_delay_seconds: 20
|
| 375 |
+
backoff_multiplier: 2
|
| 376 |
+
jitter_ratio: 0.2
|
| 377 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 378 |
+
rate_limits:
|
| 379 |
+
max_concurrency: 2
|
| 380 |
+
supports_system_prompt: true
|
| 381 |
+
supports_stream: true
|
| 382 |
+
supports_reasoning: true
|
| 383 |
+
provider_kwargs: {}
|
| 384 |
+
- model_id: gemini-3.1-pro-preview
|
| 385 |
+
provider_name: openai_compatible
|
| 386 |
+
model_name: gemini-3.1-pro-preview
|
| 387 |
+
api_base: https://api.innospark.cn/v1
|
| 388 |
+
api_key_env: INNOSPARK_RELAY_API_KEY
|
| 389 |
+
temperature: 0
|
| 390 |
+
max_tokens: 8192
|
| 391 |
+
timeout: 240
|
| 392 |
+
retry:
|
| 393 |
+
max_attempts: 2
|
| 394 |
+
initial_delay_seconds: 1
|
| 395 |
+
max_delay_seconds: 20
|
| 396 |
+
backoff_multiplier: 2
|
| 397 |
+
jitter_ratio: 0.2
|
| 398 |
+
rate_limits:
|
| 399 |
+
max_concurrency: 100
|
| 400 |
+
supports_system_prompt: true
|
| 401 |
+
supports_stream: false
|
| 402 |
+
supports_reasoning: true
|
| 403 |
+
provider_kwargs: {}
|
| 404 |
+
- model_id: innospark-235b
|
| 405 |
+
provider_name: openai_compatible
|
| 406 |
+
model_name: InnoSpark-235B
|
| 407 |
+
api_base: https://innospark-api.aiecnu.net/v1
|
| 408 |
+
api_key_env: INNOSPARK_AIECNU_API_KEY
|
| 409 |
+
temperature: 0
|
| 410 |
+
max_tokens: 1024
|
| 411 |
+
timeout: 600
|
| 412 |
+
retry:
|
| 413 |
+
max_attempts: 2
|
| 414 |
+
initial_delay_seconds: 5
|
| 415 |
+
max_delay_seconds: 120
|
| 416 |
+
backoff_multiplier: 2
|
| 417 |
+
jitter_ratio: 0.2
|
| 418 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 419 |
+
rate_limits:
|
| 420 |
+
max_concurrency: 1
|
| 421 |
+
supports_system_prompt: true
|
| 422 |
+
supports_stream: false
|
| 423 |
+
supports_reasoning: false
|
| 424 |
+
provider_kwargs:
|
| 425 |
+
runtime_max_tokens: 512
|
| 426 |
+
- model_id: qwen3.5-397b-a17b
|
| 427 |
+
provider_name: openai_compatible
|
| 428 |
+
model_name: qwen3.5-397b-a17b
|
| 429 |
+
api_base: https://dashscope.aliyuncs.com/compatible-mode/v1
|
| 430 |
+
api_key_env: DASHSCOPE_API_KEY
|
| 431 |
+
temperature: 0
|
| 432 |
+
max_tokens: 8192
|
| 433 |
+
timeout: 240
|
| 434 |
+
retry:
|
| 435 |
+
max_attempts: 3
|
| 436 |
+
initial_delay_seconds: 1
|
| 437 |
+
max_delay_seconds: 20
|
| 438 |
+
backoff_multiplier: 2
|
| 439 |
+
jitter_ratio: 0.2
|
| 440 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 441 |
+
rate_limits:
|
| 442 |
+
max_concurrency: 10
|
| 443 |
+
supports_system_prompt: true
|
| 444 |
+
supports_stream: false
|
| 445 |
+
supports_reasoning: false
|
| 446 |
+
provider_kwargs:
|
| 447 |
+
# DashScope only honors top-level enable_thinking for this model;
|
| 448 |
+
# chat_template_kwargs.enable_thinking is IGNORED here (verified empirically),
|
| 449 |
+
# so we must send it at the top level. provider_kwargs is merged into the
|
| 450 |
+
# request payload top level by the openai_compatible adapter.
|
| 451 |
+
enable_thinking: false
|
| 452 |
+
- model_id: safe-innospark
|
| 453 |
+
provider_name: openai_compatible
|
| 454 |
+
model_name: Safe-InnoSpark
|
| 455 |
+
api_base: https://innospark-api.aiecnu.net/v1
|
| 456 |
+
api_key_env: INNOSPARK_AIECNU_API_KEY
|
| 457 |
+
temperature: 0
|
| 458 |
+
max_tokens: 1024
|
| 459 |
+
timeout: 600
|
| 460 |
+
retry:
|
| 461 |
+
max_attempts: 2
|
| 462 |
+
initial_delay_seconds: 5
|
| 463 |
+
max_delay_seconds: 120
|
| 464 |
+
backoff_multiplier: 2
|
| 465 |
+
jitter_ratio: 0.2
|
| 466 |
+
retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
|
| 467 |
+
rate_limits:
|
| 468 |
+
max_concurrency: 1
|
| 469 |
+
supports_system_prompt: true
|
| 470 |
+
supports_stream: false
|
| 471 |
+
supports_reasoning: false
|
| 472 |
+
provider_kwargs:
|
| 473 |
+
runtime_max_tokens: 512
|
configs/module_registry.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
modules:
|
| 2 |
+
- module_name: 安全可信
|
| 3 |
+
enabled: true
|
| 4 |
+
description: 教育安全与可信相关评测。
|
| 5 |
+
owner: current
|
| 6 |
+
- module_name: 高阶育人
|
| 7 |
+
enabled: true
|
| 8 |
+
description: 教育场景中的高阶育人能力评测。
|
| 9 |
+
owner: current
|
| 10 |
+
- module_name: 通用模型
|
| 11 |
+
enabled: true
|
| 12 |
+
description: 通用模型能力 benchmark。
|
| 13 |
+
owner: current
|
| 14 |
+
- module_name: 基本教育
|
| 15 |
+
enabled: true
|
| 16 |
+
description: 基本教育多场景评测(45题),运行时使用内部 basic_education_runtime 多轮执行模块。
|
| 17 |
+
owner: current
|
configs/providers.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
providers:
|
| 2 |
+
- provider_name: mock
|
| 3 |
+
adapter: mock
|
| 4 |
+
capabilities:
|
| 5 |
+
supports_system_prompt: true
|
| 6 |
+
supports_stream: false
|
| 7 |
+
supports_reasoning: false
|
| 8 |
+
default_timeout: 10
|
| 9 |
+
rate_limits:
|
| 10 |
+
max_concurrency: 100
|
| 11 |
+
- provider_name: openai_compatible
|
| 12 |
+
adapter: openai_compatible
|
| 13 |
+
capabilities:
|
| 14 |
+
supports_system_prompt: true
|
| 15 |
+
supports_stream: true
|
| 16 |
+
supports_reasoning: true
|
| 17 |
+
default_timeout: 60
|
| 18 |
+
endpoint_path: /chat/completions
|
| 19 |
+
headers: {}
|
| 20 |
+
rate_limits:
|
| 21 |
+
max_concurrency: 100
|
dashboard/README.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hosted ELBench runner
|
| 2 |
+
|
| 3 |
+
Visitors bring an OpenRouter key, pick a model, and run a subset of ELBench.
|
| 4 |
+
The Space process calls `BenchmarkRunner`. Prompts, retries, and scores are the
|
| 5 |
+
ones that runner already uses.
|
| 6 |
+
|
| 7 |
+
The Hugging Face config is the YAML block at the top of the repository
|
| 8 |
+
[README.md](../README.md), not this file. Spaces read `README.md` at the
|
| 9 |
+
repository root. `app_file` is `dashboard/app.py`. `python_version` is `3.12`
|
| 10 |
+
because the package requires Python 3.11 or newer and the Space default is
|
| 11 |
+
3.10. `sdk_version` is `6.4.0`, the Gradio version this app was run against.
|
| 12 |
+
`suggested_hardware: cpu-basic` is only a suggestion for someone duplicating
|
| 13 |
+
the Space. It does not assign hardware.
|
| 14 |
+
|
| 15 |
+
## Host
|
| 16 |
+
|
| 17 |
+
Checked against the Spaces overview, configuration reference, and GPU docs:
|
| 18 |
+
|
| 19 |
+
- CPU Basic is 2 vCPU, 16 GB RAM, and 50 GB of disk, and its hourly price is free.
|
| 20 |
+
- Creating a Gradio or Docker Space requires a paid plan: PRO for a personal account, Team or Enterprise for an organization. Static Spaces are the ones that are free for every account. A free personal account can host up to two Gradio Spaces on ZeroGPU. ZeroGPU is not this app.
|
| 21 |
+
- CPU Basic sleeps after 48 hours without a visit. A visit starts it again. Custom sleep times require paid hardware.
|
| 22 |
+
- The disk is not persistent. A restart drops `outputs/`.
|
| 23 |
+
- Dependencies come from the root `requirements.txt`. Judge credentials belong in Space Settings as secrets. Gradio Spaces expose those secrets as environment variables. Set `QWEN_TEST_API_KEY` there for the official judge. Do not point judged tasks at a different model.
|
| 24 |
+
- `demo.queue(default_concurrency_limit=1)` runs at import. The Space runtime launches the imported demo and does not execute the `if __name__` block, so the one-run limit has to be set on the object itself.
|
| 25 |
+
|
| 26 |
+
`高阶育人-edu.jsonl` is scored by the rule judge, and preflight still requires the judge key. That hold is kept.
|
| 27 |
+
|
| 28 |
+
## OpenRouter
|
| 29 |
+
|
| 30 |
+
Checked against the quickstart, chat-completions reference, models list, provider routing, FAQ, and limits pages, plus a live `GET /api/v1/models`:
|
| 31 |
+
|
| 32 |
+
- Requests are `POST https://openrouter.ai/api/v1/chat/completions` with `Authorization: Bearer`. That matches `configs/providers.yaml` `endpoint_path: /chat/completions`.
|
| 33 |
+
- The model field is the slug the visitor selected, including a `:nitro` or `:floor` suffix. Those two suffixes are routing variants, not catalog rows. `:free` and `:batch` are catalog rows and are not rewritten.
|
| 34 |
+
- `pricing.prompt` and `pricing.completion` are USD per token, returned as strings. The live catalog had no `pricing.request` field. A negative price is treated as unknown.
|
| 35 |
+
- The model list answers without a key. If the typed key is rejected, the loader retries the public list.
|
| 36 |
+
- Default routing load-balances providers and falls back to another provider of the same slug. Unsupported sampling parameters are ignored unless `require_parameters` is set. This app does not send a `provider` object, and the estimate says so.
|
| 37 |
+
- A `:free` slug is capped at 20 requests per minute, and at 50 per day until the account has purchased at least 10 credits, then 1,000 per day. Paid slugs are limited by credits, the in-flight spending budget, the upstream provider, and DDoS protection.
|
| 38 |
+
- `HTTP-Referer` and `X-OpenRouter-Title` are optional. They attribute the app on OpenRouter rankings. They are not required for the call, and the client does not add them.
|
| 39 |
+
|
| 40 |
+
## What the visitor can change
|
| 41 |
+
|
| 42 |
+
- OpenRouter key and model id
|
| 43 |
+
- modules, source files, dimensions, and `max_samples`
|
| 44 |
+
- concurrency from 1 to 100 (the hosted default is 8; `configs/app.yaml` is 100)
|
| 45 |
+
|
| 46 |
+
Temperature, max tokens, streaming, provider kwargs, prompts, and judges are not form fields.
|
| 47 |
+
|
| 48 |
+
## Key handling
|
| 49 |
+
|
| 50 |
+
The key is held in `ELBENCH_VISITOR_OPENROUTER_API_KEY` for the duration of the estimate or run, then removed. It is not written to `configs/` or `key.txt`. The basic-education executor does write it into `outputs/basic_education/<run_id>/.../basic_education_runtime.yaml`, because that runtime reads the key from the rendered config. The app deletes that run directory when the run returns, including on failure. A killed process can leave the file on disk.
|
| 51 |
+
|
| 52 |
+
## Local use
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
pip install -e ".[basic-education,dashboard]"
|
| 56 |
+
python dashboard/app.py
|
| 57 |
+
```
|
dashboard/app.py
ADDED
|
@@ -0,0 +1,1405 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ELBench-live: the published ELBench results, and a hosted runner for any OpenRouter model.
|
| 2 |
+
|
| 3 |
+
Runs go through ``BenchmarkRunner``, ``OpenAICompatibleClient``, and ``JudgeRouter``, so
|
| 4 |
+
prompts, retries, and scoring are the benchmark's own.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import sys
|
| 10 |
+
import threading
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Any
|
| 14 |
+
|
| 15 |
+
try:
|
| 16 |
+
import spaces # type: ignore[import-not-found]
|
| 17 |
+
except ImportError:
|
| 18 |
+
spaces = None # type: ignore[assignment]
|
| 19 |
+
|
| 20 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 21 |
+
SRC = ROOT / "src"
|
| 22 |
+
if str(SRC) not in sys.path:
|
| 23 |
+
sys.path.insert(0, str(SRC))
|
| 24 |
+
|
| 25 |
+
import gradio as gr
|
| 26 |
+
import httpx
|
| 27 |
+
|
| 28 |
+
from elbench.dashboard.catalog import Catalog, CatalogFile, load_catalog
|
| 29 |
+
from elbench.dashboard.english import AREA_LABELS, area_label, english_dimension, plain_file
|
| 30 |
+
from elbench.dashboard.leaderboard import (
|
| 31 |
+
LeaderboardStore,
|
| 32 |
+
build_comparison_choices,
|
| 33 |
+
build_comparison_table,
|
| 34 |
+
build_pareto_frontier,
|
| 35 |
+
custom_leaderboard_rows,
|
| 36 |
+
load_official_models,
|
| 37 |
+
official_leaderboard_rows,
|
| 38 |
+
official_per_file_rows,
|
| 39 |
+
pareto_cost_metric_choices,
|
| 40 |
+
pareto_score_metric_choices,
|
| 41 |
+
seed_official_openrouter_prices,
|
| 42 |
+
)
|
| 43 |
+
from elbench.dashboard.model_sizes import ModelSize, load_sizes, model_size
|
| 44 |
+
from elbench.dashboard.plan import Plan, build_plan, render_plan
|
| 45 |
+
from elbench.dashboard.protocol import DEFAULT_SYSTEM_ONE_MODELS, catalog_slug
|
| 46 |
+
from elbench.dashboard.run import (
|
| 47 |
+
execute_run,
|
| 48 |
+
failures_progress_path,
|
| 49 |
+
judged_progress_path,
|
| 50 |
+
make_run_id,
|
| 51 |
+
public_error,
|
| 52 |
+
visitor_api_key,
|
| 53 |
+
)
|
| 54 |
+
from elbench.utils.spend import ROLE_LABELS, SpendLedger, SpendLimits
|
| 55 |
+
|
| 56 |
+
OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
|
| 57 |
+
OPENROUTER_DECISION_MODELS_URL = "https://openrouter.ai/api/v1/models?output_modalities=decisions"
|
| 58 |
+
OPENROUTER_KEY_URL = "https://openrouter.ai/api/v1/key"
|
| 59 |
+
OPENROUTER_KEYS_PAGE = "https://openrouter.ai/settings/keys"
|
| 60 |
+
DEFAULT_TARGET_SLUG = "openai/gpt-5.4"
|
| 61 |
+
DEFAULT_CHAT_JUDGE_SLUG = "qwen/qwen3-235b-a22b-2507"
|
| 62 |
+
DEFAULT_SYSTEM_ONE_JUDGE_SLUG = "liquid/d1-ultra"
|
| 63 |
+
DEFAULT_MULTITURN_CHAT_SLUG = "liquid/lfm-2.5-24b-a2b"
|
| 64 |
+
DEFAULT_BUDGET_USD = 5.0
|
| 65 |
+
|
| 66 |
+
ELBENCH_PAPER_URL = "https://arxiv.org/abs/2608.09548"
|
| 67 |
+
ELBENCH_GITHUB_URL = "https://github.com/ZeroLoss-Lab/ELBench"
|
| 68 |
+
AI_FOR_EDUCATION_LEADERBOARD_URL = "https://benchmarks.ai-for-education.org/"
|
| 69 |
+
ELBENCH_DATASET_URL = "https://huggingface.co/datasets/ZeroLoss-Lab/ELBench"
|
| 70 |
+
ELBENCH_RESULTS_URL = "https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results"
|
| 71 |
+
|
| 72 |
+
PRESET_QUICK = "quick"
|
| 73 |
+
PRESET_AUTO = "auto"
|
| 74 |
+
PRESET_FULL = "full"
|
| 75 |
+
QUICK_CHECK_ITEMS = 5
|
| 76 |
+
|
| 77 |
+
# Filter sliders: the right-hand end means "no limit".
|
| 78 |
+
PRICE_FILTER_MAX = 30.0
|
| 79 |
+
SIZE_FILTER_MAX = 1000.0
|
| 80 |
+
MODEL_SIZES = load_sizes()
|
| 81 |
+
|
| 82 |
+
FOCUS_CHOICES = [(label, key) for label, key in pareto_score_metric_choices() if not key.startswith("file:")]
|
| 83 |
+
TASK_CHOICES = [("None, use the area above", "")] + [
|
| 84 |
+
(label, key) for label, key in pareto_score_metric_choices() if key.startswith("file:")
|
| 85 |
+
]
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _load_catalog() -> Catalog | None:
|
| 89 |
+
try:
|
| 90 |
+
return load_catalog(ROOT / "configs")
|
| 91 |
+
except Exception as exc: # noqa: BLE001
|
| 92 |
+
_load_catalog.error = str(exc) # type: ignore[attr-defined]
|
| 93 |
+
return None
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
_load_catalog.error = "" # type: ignore[attr-defined]
|
| 97 |
+
CATALOG = _load_catalog()
|
| 98 |
+
LEADERBOARD_STORE = LeaderboardStore(ROOT)
|
| 99 |
+
try:
|
| 100 |
+
LEADERBOARD_STORE.sync_from_hub()
|
| 101 |
+
except Exception: # noqa: BLE001
|
| 102 |
+
pass
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# ---------------------------------------------------------------------------
|
| 106 |
+
# OpenRouter catalog and prices (one cached snapshot per process)
|
| 107 |
+
# ---------------------------------------------------------------------------
|
| 108 |
+
|
| 109 |
+
CATALOG_SNAPSHOT_TTL_SECONDS = 6 * 3600
|
| 110 |
+
_SNAPSHOT: dict[str, Any] = {"value": None, "fetched_at": 0.0}
|
| 111 |
+
_SNAPSHOT_LOCK = threading.Lock()
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _nonnegative_price(value) -> float | None:
|
| 115 |
+
if value is None or value == "":
|
| 116 |
+
return None
|
| 117 |
+
try:
|
| 118 |
+
price = float(value)
|
| 119 |
+
except (TypeError, ValueError):
|
| 120 |
+
return None
|
| 121 |
+
if price < 0:
|
| 122 |
+
return None
|
| 123 |
+
return price
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _fetch_model_catalog(api_key: str, url: str = OPENROUTER_MODELS_URL, timeout: float = 45) -> dict:
|
| 127 |
+
headers = {"Accept": "application/json"}
|
| 128 |
+
secret = (api_key or "").strip()
|
| 129 |
+
if secret:
|
| 130 |
+
headers["Authorization"] = f"Bearer {secret}"
|
| 131 |
+
response = httpx.get(url, headers=headers, timeout=timeout)
|
| 132 |
+
response.raise_for_status()
|
| 133 |
+
payload = response.json()
|
| 134 |
+
if not isinstance(payload, dict):
|
| 135 |
+
raise ValueError("OpenRouter /api/v1/models did not return an object.")
|
| 136 |
+
return payload
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _per_million(price: float | None) -> str:
|
| 140 |
+
if price is None:
|
| 141 |
+
return "?"
|
| 142 |
+
value = price * 1_000_000
|
| 143 |
+
return f"${value:.2f}" if value >= 0.1 or value == 0 else f"${value:.3f}"
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _parse_catalog_rows(payload: dict) -> tuple[list[dict[str, Any]], dict[str, tuple[float | None, float | None]]]:
|
| 147 |
+
models: list[dict[str, Any]] = []
|
| 148 |
+
prices: dict[str, tuple[float | None, float | None]] = {}
|
| 149 |
+
for item in payload.get("data") or []:
|
| 150 |
+
model_id = str(item.get("id") or "").strip()
|
| 151 |
+
if not model_id:
|
| 152 |
+
continue
|
| 153 |
+
pricing = item.get("pricing") or {}
|
| 154 |
+
prompt = _nonnegative_price(pricing.get("prompt"))
|
| 155 |
+
completion = _nonnegative_price(pricing.get("completion"))
|
| 156 |
+
prices[model_id] = (prompt, completion)
|
| 157 |
+
models.append(
|
| 158 |
+
{
|
| 159 |
+
"id": model_id,
|
| 160 |
+
"name": str(item.get("name") or model_id),
|
| 161 |
+
"prompt": prompt,
|
| 162 |
+
"completion": completion,
|
| 163 |
+
"hugging_face_id": item.get("hugging_face_id") or None,
|
| 164 |
+
}
|
| 165 |
+
)
|
| 166 |
+
models.sort(key=lambda m: m["name"].lower())
|
| 167 |
+
return models, prices
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _model_choices(models: list[dict[str, Any]]) -> list[tuple[str, str]]:
|
| 171 |
+
choices = []
|
| 172 |
+
for m in models:
|
| 173 |
+
if m["prompt"] is None:
|
| 174 |
+
price = "no price listed"
|
| 175 |
+
else:
|
| 176 |
+
price = f"{_per_million(m['prompt'])} in / {_per_million(m['completion'])} out per 1M tokens"
|
| 177 |
+
size = model_size(m["id"], m.get("hugging_face_id"), MODEL_SIZES)
|
| 178 |
+
choices.append((f"{m['name']} · {price}" + (f" · {size.label()}" if size else ""), m["id"]))
|
| 179 |
+
return choices
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _blended_per_1m(prompt: float | None, completion: float | None) -> float | None:
|
| 183 |
+
"""The chart's default price measure: 3 parts input to 1 part output, per million tokens."""
|
| 184 |
+
if prompt is None or completion is None:
|
| 185 |
+
return None
|
| 186 |
+
return (3 * prompt + completion) / 4 * 1_000_000
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def passes_filters(
|
| 190 |
+
price_per_1m: float | None, size: ModelSize | None, max_price, max_size, include_undisclosed
|
| 191 |
+
) -> bool:
|
| 192 |
+
if max_price is not None and float(max_price) < PRICE_FILTER_MAX:
|
| 193 |
+
if price_per_1m is None or price_per_1m > float(max_price):
|
| 194 |
+
return False
|
| 195 |
+
if size is None:
|
| 196 |
+
return bool(include_undisclosed)
|
| 197 |
+
return max_size is None or float(max_size) >= SIZE_FILTER_MAX or size.total_b <= float(max_size)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _size_of(slug: str, snapshot: dict[str, Any]) -> ModelSize | None:
|
| 201 |
+
hf_ids = snapshot.get("hf_ids") or {}
|
| 202 |
+
return model_size(slug, hf_ids.get(slug) or hf_ids.get(catalog_slug(slug)), MODEL_SIZES)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _filtered_chat_models(snapshot: dict[str, Any], max_price, max_size, include_undisclosed) -> list[dict[str, Any]]:
|
| 206 |
+
return [
|
| 207 |
+
m
|
| 208 |
+
for m in snapshot["chat_models"]
|
| 209 |
+
if passes_filters(
|
| 210 |
+
_blended_per_1m(m["prompt"], m["completion"]),
|
| 211 |
+
model_size(m["id"], m.get("hugging_face_id"), MODEL_SIZES),
|
| 212 |
+
max_price,
|
| 213 |
+
max_size,
|
| 214 |
+
include_undisclosed,
|
| 215 |
+
)
|
| 216 |
+
]
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def _default_system_one_models() -> list[dict[str, Any]]:
|
| 220 |
+
return [
|
| 221 |
+
{"id": slug, "name": name, "prompt": price, "completion": 0.0, "hugging_face_id": None}
|
| 222 |
+
for name, slug, price in DEFAULT_SYSTEM_ONE_MODELS
|
| 223 |
+
]
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _default_prices() -> dict[str, tuple[float | None, float | None]]:
|
| 227 |
+
table: dict[str, tuple[float | None, float | None]] = seed_official_openrouter_prices()
|
| 228 |
+
for _, slug, prompt_price in DEFAULT_SYSTEM_ONE_MODELS:
|
| 229 |
+
table[slug] = (prompt_price, 0.0)
|
| 230 |
+
return table
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def openrouter_snapshot() -> dict[str, Any]:
|
| 234 |
+
"""Chat models, decision models, and prices from OpenRouter, cached for a few hours."""
|
| 235 |
+
with _SNAPSHOT_LOCK:
|
| 236 |
+
cached = _SNAPSHOT["value"]
|
| 237 |
+
if cached is not None and time.time() - _SNAPSHOT["fetched_at"] < CATALOG_SNAPSHOT_TTL_SECONDS:
|
| 238 |
+
return cached
|
| 239 |
+
prices = _default_prices()
|
| 240 |
+
try:
|
| 241 |
+
chat_models, live_prices = _parse_catalog_rows(_fetch_model_catalog("", OPENROUTER_MODELS_URL, timeout=15))
|
| 242 |
+
except Exception: # noqa: BLE001
|
| 243 |
+
if cached is not None:
|
| 244 |
+
return cached
|
| 245 |
+
return {
|
| 246 |
+
"hf_ids": {},
|
| 247 |
+
"chat_models": [],
|
| 248 |
+
"decision_models": _default_system_one_models(),
|
| 249 |
+
"prices": prices,
|
| 250 |
+
"label": "Prices stored with the published results (OpenRouter could not be reached).",
|
| 251 |
+
"live": False,
|
| 252 |
+
}
|
| 253 |
+
prices.update(live_prices)
|
| 254 |
+
decision_models = _default_system_one_models()
|
| 255 |
+
try:
|
| 256 |
+
fetched, decision_prices = _parse_catalog_rows(
|
| 257 |
+
_fetch_model_catalog("", OPENROUTER_DECISION_MODELS_URL, timeout=15)
|
| 258 |
+
)
|
| 259 |
+
if fetched:
|
| 260 |
+
decision_models = fetched
|
| 261 |
+
prices.update(decision_prices)
|
| 262 |
+
except Exception: # noqa: BLE001
|
| 263 |
+
pass
|
| 264 |
+
value = {
|
| 265 |
+
"hf_ids": {m["id"]: m["hugging_face_id"] for m in chat_models if m["hugging_face_id"]},
|
| 266 |
+
"chat_models": chat_models,
|
| 267 |
+
"decision_models": decision_models,
|
| 268 |
+
"prices": prices,
|
| 269 |
+
"label": f"Prices from OpenRouter, {time.strftime('%d %b %Y %H:%M UTC', time.gmtime())}.",
|
| 270 |
+
"live": True,
|
| 271 |
+
}
|
| 272 |
+
_SNAPSHOT.update(value=value, fetched_at=time.time())
|
| 273 |
+
return value
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def price_snapshot() -> tuple[dict[str, tuple[float | None, float | None]], str]:
|
| 277 |
+
snapshot = openrouter_snapshot()
|
| 278 |
+
return dict(snapshot["prices"]), snapshot["label"]
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _price_pair(prices: dict, model_id: str) -> tuple[float | None, float | None]:
|
| 282 |
+
key = str(model_id or "").strip()
|
| 283 |
+
raw = (prices or {}).get(key) or (prices or {}).get(catalog_slug(key))
|
| 284 |
+
if not raw:
|
| 285 |
+
return None, None
|
| 286 |
+
return raw[0], raw[1]
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _pick_preferred(choices: list[tuple[str, str]], preferred: str) -> str:
|
| 290 |
+
slugs = [slug for _, slug in choices]
|
| 291 |
+
if preferred in slugs or not slugs:
|
| 292 |
+
return preferred
|
| 293 |
+
return slugs[0]
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def check_key(api_key: str) -> tuple[bool, str]:
|
| 297 |
+
"""Free call to OpenRouter's key endpoint: is the key valid, and how much credit is left."""
|
| 298 |
+
try:
|
| 299 |
+
response = httpx.get(OPENROUTER_KEY_URL, headers={"Authorization": f"Bearer {api_key}"}, timeout=15)
|
| 300 |
+
except Exception: # noqa: BLE001
|
| 301 |
+
return True, "Could not check the key with OpenRouter just now; the run will tell you if it is rejected."
|
| 302 |
+
if response.status_code in {401, 403}:
|
| 303 |
+
return False, "OpenRouter did not accept this API key. Check that it was copied in full."
|
| 304 |
+
if response.status_code != 200:
|
| 305 |
+
return True, "Could not check the key with OpenRouter just now; the run will tell you if it is rejected."
|
| 306 |
+
data = (response.json() or {}).get("data") or {}
|
| 307 |
+
remaining = data.get("limit_remaining")
|
| 308 |
+
if isinstance(remaining, (int, float)):
|
| 309 |
+
return True, f"Key accepted. This key has ${remaining:,.2f} of credit left."
|
| 310 |
+
return True, "Key accepted."
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
# ---------------------------------------------------------------------------
|
| 314 |
+
# Run scope
|
| 315 |
+
# ---------------------------------------------------------------------------
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def _module_choices() -> list[tuple[str, str]]:
|
| 319 |
+
if CATALOG is None:
|
| 320 |
+
return []
|
| 321 |
+
return [(area_label(m), m) for m in CATALOG.modules()]
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def _file_label(entry: CatalogFile) -> str:
|
| 325 |
+
grading = "graded by the judge model" if entry.scoring_mode in {"llm", "runtime"} else "checked automatically"
|
| 326 |
+
return f"{area_label(entry.module)} · {plain_file(entry.canonical_name)} · {entry.item_count} questions, {grading}"
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _file_choices(files: list[CatalogFile]) -> list[tuple[str, str]]:
|
| 330 |
+
return [(_file_label(entry), entry.canonical_name) for entry in files]
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _dimension_choices(files: list[CatalogFile]) -> list[tuple[str, str]]:
|
| 334 |
+
seen: list[str] = []
|
| 335 |
+
for entry in files:
|
| 336 |
+
for dimension in entry.dimensions:
|
| 337 |
+
if dimension not in seen:
|
| 338 |
+
seen.append(dimension)
|
| 339 |
+
return [(english_dimension(d), d) for d in seen]
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _selected_entries(names: list[str] | None) -> list[CatalogFile]:
|
| 343 |
+
if CATALOG is None:
|
| 344 |
+
return []
|
| 345 |
+
wanted = set(names or [])
|
| 346 |
+
return [entry for entry in CATALOG.files if entry.canonical_name in wanted]
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def _preset_choices() -> list[tuple[str, str]]:
|
| 350 |
+
if CATALOG is None:
|
| 351 |
+
return []
|
| 352 |
+
auto_items = sum(entry.item_count for entry in CATALOG.default_files())
|
| 353 |
+
full_items = sum(entry.item_count for entry in CATALOG.files)
|
| 354 |
+
return [
|
| 355 |
+
(f"Quick check · {QUICK_CHECK_ITEMS} questions, to see that everything works", PRESET_QUICK),
|
| 356 |
+
(f"Automatically checked set · {auto_items:,} questions, no judge model needed", PRESET_AUTO),
|
| 357 |
+
(f"Full benchmark · all {full_items:,} questions · appears on the leaderboard chart", PRESET_FULL),
|
| 358 |
+
]
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def apply_preset(preset: str):
|
| 362 |
+
if CATALOG is None:
|
| 363 |
+
return gr.update(), gr.update(), gr.update(), gr.update()
|
| 364 |
+
if preset == PRESET_FULL:
|
| 365 |
+
files = list(CATALOG.files)
|
| 366 |
+
modules = CATALOG.modules()
|
| 367 |
+
cap = None
|
| 368 |
+
else:
|
| 369 |
+
files = CATALOG.default_files()
|
| 370 |
+
modules = CATALOG.default_modules()
|
| 371 |
+
cap = QUICK_CHECK_ITEMS if preset == PRESET_QUICK else None
|
| 372 |
+
return (
|
| 373 |
+
gr.update(value=modules),
|
| 374 |
+
gr.update(value=[entry.canonical_name for entry in files]),
|
| 375 |
+
gr.update(choices=_dimension_choices(files), value=[]),
|
| 376 |
+
gr.update(value=cap),
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
def on_modules(modules: list[str]):
|
| 381 |
+
if CATALOG is None:
|
| 382 |
+
return gr.update(), gr.update()
|
| 383 |
+
chosen = CATALOG.files_in_modules(set(modules or []))
|
| 384 |
+
return (
|
| 385 |
+
gr.update(value=[entry.canonical_name for entry in chosen]),
|
| 386 |
+
gr.update(choices=_dimension_choices(chosen), value=[]),
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
|
| 390 |
+
def on_files(files: list[str]):
|
| 391 |
+
return gr.update(choices=_dimension_choices(_selected_entries(files)), value=[])
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _max_samples(value) -> int | None:
|
| 395 |
+
if value is None or value == "":
|
| 396 |
+
return None
|
| 397 |
+
return int(value)
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def _budget(value) -> float | None:
|
| 401 |
+
if value is None or value == "":
|
| 402 |
+
return None
|
| 403 |
+
try:
|
| 404 |
+
number = float(value)
|
| 405 |
+
except (TypeError, ValueError):
|
| 406 |
+
return None
|
| 407 |
+
return number if number > 0 else None
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
# ---------------------------------------------------------------------------
|
| 411 |
+
# Model pickers
|
| 412 |
+
# ---------------------------------------------------------------------------
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _models_shown_note(shown: int, total: int) -> str:
|
| 416 |
+
return f"Showing {shown:,} of {total:,} OpenRouter models." if total else ""
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def load_model_pickers(use_system_one: bool, max_price, max_size, include_undisclosed):
|
| 420 |
+
snapshot = openrouter_snapshot()
|
| 421 |
+
chat = _model_choices(snapshot["chat_models"])
|
| 422 |
+
targets = _model_choices(_filtered_chat_models(snapshot, max_price, max_size, include_undisclosed))
|
| 423 |
+
judge_choices = _model_choices(snapshot["decision_models"]) if use_system_one else chat
|
| 424 |
+
judge_default = DEFAULT_SYSTEM_ONE_JUDGE_SLUG if use_system_one else DEFAULT_CHAT_JUDGE_SLUG
|
| 425 |
+
return (
|
| 426 |
+
gr.update(choices=targets, value=_pick_preferred(targets, DEFAULT_TARGET_SLUG)),
|
| 427 |
+
gr.update(choices=judge_choices, value=_pick_preferred(judge_choices, judge_default)),
|
| 428 |
+
gr.update(choices=chat, value=_pick_preferred(chat, DEFAULT_MULTITURN_CHAT_SLUG)),
|
| 429 |
+
dict(snapshot["prices"]),
|
| 430 |
+
_models_shown_note(len(targets), len(chat)),
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def filter_target_models(current, max_price, max_size, include_undisclosed):
|
| 435 |
+
snapshot = openrouter_snapshot()
|
| 436 |
+
targets = _model_choices(_filtered_chat_models(snapshot, max_price, max_size, include_undisclosed))
|
| 437 |
+
slugs = [slug for _, slug in targets]
|
| 438 |
+
value = current if current in slugs else None
|
| 439 |
+
return (
|
| 440 |
+
gr.update(choices=targets, value=value),
|
| 441 |
+
_models_shown_note(len(targets), len(snapshot["chat_models"])),
|
| 442 |
+
)
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def on_toggle_system_one(use_system_one: bool):
|
| 446 |
+
snapshot = openrouter_snapshot()
|
| 447 |
+
if use_system_one:
|
| 448 |
+
choices = _model_choices(snapshot["decision_models"])
|
| 449 |
+
return gr.update(
|
| 450 |
+
label="Judge (decision model)",
|
| 451 |
+
choices=choices,
|
| 452 |
+
value=_pick_preferred(choices, DEFAULT_SYSTEM_ONE_JUDGE_SLUG),
|
| 453 |
+
info="Decision models return a verdict with no written output, which is much cheaper per answer.",
|
| 454 |
+
)
|
| 455 |
+
choices = _model_choices(snapshot["chat_models"])
|
| 456 |
+
return gr.update(
|
| 457 |
+
label="Judge model",
|
| 458 |
+
choices=choices,
|
| 459 |
+
value=_pick_preferred(choices, DEFAULT_CHAT_JUDGE_SLUG),
|
| 460 |
+
info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's judge.",
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
def on_toggle_multiturn(use_multiturn: bool):
|
| 465 |
+
return gr.update(visible=bool(use_multiturn), interactive=bool(use_multiturn))
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
# ---------------------------------------------------------------------------
|
| 469 |
+
# Estimate and run
|
| 470 |
+
# ---------------------------------------------------------------------------
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def _usd(value: float | None) -> str:
|
| 474 |
+
if value is None:
|
| 475 |
+
return "?"
|
| 476 |
+
if value >= 0.995:
|
| 477 |
+
return f"${value:,.2f}"
|
| 478 |
+
if value >= 0.01:
|
| 479 |
+
return f"${value:.2f}"
|
| 480 |
+
if value == 0:
|
| 481 |
+
return "$0"
|
| 482 |
+
return "$" + f"{value:.6f}".rstrip("0")
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
def _make_plan(
|
| 486 |
+
*,
|
| 487 |
+
model_id,
|
| 488 |
+
judge_model_id,
|
| 489 |
+
use_system_one_judge,
|
| 490 |
+
use_multiturn_chat,
|
| 491 |
+
multiturn_model_id,
|
| 492 |
+
modules,
|
| 493 |
+
files,
|
| 494 |
+
dimensions,
|
| 495 |
+
max_samples,
|
| 496 |
+
concurrency,
|
| 497 |
+
prices,
|
| 498 |
+
) -> Plan:
|
| 499 |
+
prompt_price, completion_price = _price_pair(prices, str(model_id).strip())
|
| 500 |
+
judge_prompt_price, judge_completion_price = _price_pair(prices, str(judge_model_id).strip())
|
| 501 |
+
mt_prompt_price, mt_completion_price = _price_pair(prices, str(multiturn_model_id or "").strip())
|
| 502 |
+
return build_plan(
|
| 503 |
+
CATALOG,
|
| 504 |
+
openrouter_model=str(model_id).strip(),
|
| 505 |
+
modules=set(modules or []),
|
| 506 |
+
source_files=set(files or []),
|
| 507 |
+
dimensions=set(dimensions or []),
|
| 508 |
+
max_samples=_max_samples(max_samples),
|
| 509 |
+
prompt_usd_per_token=prompt_price,
|
| 510 |
+
completion_usd_per_token=completion_price,
|
| 511 |
+
max_concurrency=int(concurrency),
|
| 512 |
+
judge_openrouter_model=str(judge_model_id).strip(),
|
| 513 |
+
use_system_one_judge=bool(use_system_one_judge),
|
| 514 |
+
judge_prompt_usd_per_token=judge_prompt_price,
|
| 515 |
+
judge_completion_usd_per_token=judge_completion_price,
|
| 516 |
+
use_multiturn_chat_model=bool(use_multiturn_chat),
|
| 517 |
+
multiturn_openrouter_model=str(multiturn_model_id or "").strip() if use_multiturn_chat else None,
|
| 518 |
+
multiturn_prompt_usd_per_token=mt_prompt_price,
|
| 519 |
+
multiturn_completion_usd_per_token=mt_completion_price,
|
| 520 |
+
)
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
def budget_problems(plan: Plan, total: float | None, per_role: dict[str, float | None]) -> list[str]:
|
| 524 |
+
"""Limits that the lowest estimate already exceeds, so the run could not finish."""
|
| 525 |
+
problems: list[str] = []
|
| 526 |
+
if total is not None and plan.low_usd is not None and plan.low_usd > total:
|
| 527 |
+
problems.append(
|
| 528 |
+
f"Your spending limit ({_usd(total)}) is below the lowest estimate for this run "
|
| 529 |
+
f"({_usd(plan.low_usd)}). Raise the limit or choose fewer questions."
|
| 530 |
+
)
|
| 531 |
+
for role, cap in per_role.items():
|
| 532 |
+
low = (plan.component_usd.get(role) or (None, None))[0]
|
| 533 |
+
if cap is not None and low is not None and low > cap:
|
| 534 |
+
problems.append(
|
| 535 |
+
f"The limit for the {ROLE_LABELS[role]} ({_usd(cap)}) is below its lowest estimate ({_usd(low)})."
|
| 536 |
+
)
|
| 537 |
+
return problems
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
def plain_estimate(
|
| 541 |
+
plan: Plan,
|
| 542 |
+
*,
|
| 543 |
+
total_budget: float | None,
|
| 544 |
+
role_budgets: dict[str, float | None],
|
| 545 |
+
key_note: str,
|
| 546 |
+
resumable: dict | None,
|
| 547 |
+
) -> str:
|
| 548 |
+
lines: list[str] = [f"*{key_note}*", ""] if key_note else []
|
| 549 |
+
errors = plan.error_messages()
|
| 550 |
+
if errors:
|
| 551 |
+
lines.append("**This run can't start yet:**")
|
| 552 |
+
lines.extend(f"- {message}" for message in errors)
|
| 553 |
+
return "\n".join(lines)
|
| 554 |
+
|
| 555 |
+
questions = plan.standard_requests + plan.basic_tasks
|
| 556 |
+
lines.append(f"### {questions:,} questions for `{plan.protocol.openrouter_model}`")
|
| 557 |
+
graded = plan.judge_calls + plan.basic_tasks
|
| 558 |
+
if graded and plan.judge_model_config is not None:
|
| 559 |
+
lines.append(
|
| 560 |
+
f"{graded:,} answers are graded by `{plan.judge_model_config.model_name}`; "
|
| 561 |
+
"the rest are checked automatically against the answer key."
|
| 562 |
+
)
|
| 563 |
+
else:
|
| 564 |
+
lines.append("Every answer is checked automatically against the answer key.")
|
| 565 |
+
lines.append("")
|
| 566 |
+
|
| 567 |
+
if plan.low_usd is None or plan.high_usd is None:
|
| 568 |
+
lines.append(
|
| 569 |
+
"**No cost estimate:** OpenRouter lists no price for this model. To run it anyway, tick "
|
| 570 |
+
"*Run even though this model has no listed price* under Advanced settings."
|
| 571 |
+
)
|
| 572 |
+
else:
|
| 573 |
+
lines.append(
|
| 574 |
+
f"**Estimated cost: {_usd(plan.low_usd)} to {_usd(plan.high_usd)}.** The low end counts the "
|
| 575 |
+
"questions only; the high end assumes every answer runs to the length limit, so real runs "
|
| 576 |
+
"land well below it."
|
| 577 |
+
)
|
| 578 |
+
for role in ("target", "judge"):
|
| 579 |
+
if role in plan.component_usd:
|
| 580 |
+
low, high = plan.component_usd[role]
|
| 581 |
+
cap = role_budgets.get(role)
|
| 582 |
+
limit = f" · limit {_usd(cap)}" if cap is not None else ""
|
| 583 |
+
lines.append(f"- {ROLE_LABELS[role].capitalize()}: {_usd(low)} to {_usd(high)}{limit}")
|
| 584 |
+
if role_budgets.get("student") is not None:
|
| 585 |
+
lines.append(f"- Simulated student: not estimated · limit {_usd(role_budgets['student'])}")
|
| 586 |
+
if plan.basic_tasks:
|
| 587 |
+
lines.append(
|
| 588 |
+
"- Teaching tasks are multi-turn conversations whose length isn't known in advance; "
|
| 589 |
+
"their judge and simulated-student calls are not in this estimate but do count toward your limits."
|
| 590 |
+
)
|
| 591 |
+
lines.append("")
|
| 592 |
+
|
| 593 |
+
problems = budget_problems(plan, total_budget, role_budgets)
|
| 594 |
+
if problems:
|
| 595 |
+
lines.append("**Spending limit too low:**")
|
| 596 |
+
lines.extend(f"- {problem}" for problem in problems)
|
| 597 |
+
elif total_budget is None:
|
| 598 |
+
lines.append("No spending limit is set. The run continues until it finishes or your key runs out of credit.")
|
| 599 |
+
elif plan.high_usd is not None and plan.high_usd <= total_budget:
|
| 600 |
+
lines.append(f"Your limit of {_usd(total_budget)} covers even the high estimate.")
|
| 601 |
+
else:
|
| 602 |
+
lines.append(
|
| 603 |
+
f"**Spending limit {_usd(total_budget)}.** If spending reaches it, the run pauses and keeps every "
|
| 604 |
+
"finished answer. Press **Run** again with a higher limit to carry on from where it stopped."
|
| 605 |
+
)
|
| 606 |
+
if total_budget is not None or any(cap is not None for cap in role_budgets.values()):
|
| 607 |
+
lines.append(
|
| 608 |
+
"Questions already sent when a limit is reached still finish, so the final total can go slightly over it."
|
| 609 |
+
)
|
| 610 |
+
lines.append("")
|
| 611 |
+
|
| 612 |
+
if plan.is_full_benchmark:
|
| 613 |
+
lines.append("This is a full run: once it finishes, it appears on the leaderboard chart next to the published models.")
|
| 614 |
+
else:
|
| 615 |
+
lines.append(
|
| 616 |
+
"This is a partial run. Its results are saved with the visitor runs, but the chart only shows "
|
| 617 |
+
"models scored on every question in an area, so the comparison stays fair."
|
| 618 |
+
)
|
| 619 |
+
if plan.protocol.openrouter_model.endswith(":free"):
|
| 620 |
+
lines.append("Free models on OpenRouter are limited to about 20 requests a minute, so this will be slow.")
|
| 621 |
+
if resumable is not None:
|
| 622 |
+
lines.append(
|
| 623 |
+
f"\nAn earlier run with these exact settings stopped after {resumable.get('total_judged', 0)} "
|
| 624 |
+
"answers. Running again picks up where it left off."
|
| 625 |
+
)
|
| 626 |
+
return "\n".join(lines)
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
def _role_budgets(budget_target, budget_judge, budget_student) -> dict[str, float | None]:
|
| 630 |
+
return {"target": _budget(budget_target), "judge": _budget(budget_judge), "student": _budget(budget_student)}
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
def estimate(
|
| 634 |
+
api_key,
|
| 635 |
+
model_id,
|
| 636 |
+
judge_model_id,
|
| 637 |
+
use_system_one_judge,
|
| 638 |
+
use_multiturn_chat,
|
| 639 |
+
multiturn_model_id,
|
| 640 |
+
modules,
|
| 641 |
+
files,
|
| 642 |
+
dimensions,
|
| 643 |
+
max_samples,
|
| 644 |
+
concurrency,
|
| 645 |
+
prices,
|
| 646 |
+
total_budget,
|
| 647 |
+
budget_target,
|
| 648 |
+
budget_judge,
|
| 649 |
+
budget_student,
|
| 650 |
+
):
|
| 651 |
+
if CATALOG is None:
|
| 652 |
+
return f"The benchmark data did not load: {_load_catalog.error}", ""
|
| 653 |
+
secret = (api_key or "").strip()
|
| 654 |
+
if not secret:
|
| 655 |
+
return f"Paste your OpenRouter API key first. You can create one at [openrouter.ai]({OPENROUTER_KEYS_PAGE}).", ""
|
| 656 |
+
if not str(model_id or "").strip():
|
| 657 |
+
return "Choose the model you want to test.", ""
|
| 658 |
+
if not str(judge_model_id or "").strip():
|
| 659 |
+
return "Choose a judge model under Advanced settings.", ""
|
| 660 |
+
key_ok, key_note = check_key(secret)
|
| 661 |
+
if not key_ok:
|
| 662 |
+
return key_note, ""
|
| 663 |
+
try:
|
| 664 |
+
with visitor_api_key(secret):
|
| 665 |
+
plan = _make_plan(
|
| 666 |
+
model_id=model_id,
|
| 667 |
+
judge_model_id=judge_model_id,
|
| 668 |
+
use_system_one_judge=use_system_one_judge,
|
| 669 |
+
use_multiturn_chat=use_multiturn_chat,
|
| 670 |
+
multiturn_model_id=multiturn_model_id,
|
| 671 |
+
modules=modules,
|
| 672 |
+
files=files,
|
| 673 |
+
dimensions=dimensions,
|
| 674 |
+
max_samples=max_samples,
|
| 675 |
+
concurrency=concurrency,
|
| 676 |
+
prices=prices,
|
| 677 |
+
)
|
| 678 |
+
except Exception as exc: # noqa: BLE001
|
| 679 |
+
return public_error(exc, secret), ""
|
| 680 |
+
resumable = LEADERBOARD_STORE.find_resumable_run(config_hash=plan.config_hash)
|
| 681 |
+
summary = plain_estimate(
|
| 682 |
+
plan,
|
| 683 |
+
total_budget=_budget(total_budget),
|
| 684 |
+
role_budgets=_role_budgets(budget_target, budget_judge, budget_student),
|
| 685 |
+
key_note=key_note,
|
| 686 |
+
resumable=resumable,
|
| 687 |
+
)
|
| 688 |
+
return summary, render_plan(plan)
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def _spent_so_far(run_id: str) -> float:
|
| 692 |
+
if CATALOG is None or not run_id:
|
| 693 |
+
return 0.0
|
| 694 |
+
path = CATALOG.config.project_root / "outputs" / "hosted" / run_id / "spend.jsonl"
|
| 695 |
+
if not path.exists():
|
| 696 |
+
return 0.0
|
| 697 |
+
return sum(SpendLedger(path, SpendLimits()).spent().values())
|
| 698 |
+
|
| 699 |
+
|
| 700 |
+
def _count_lines(path: Path) -> int:
|
| 701 |
+
if not path.exists():
|
| 702 |
+
return 0
|
| 703 |
+
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
| 704 |
+
return sum(1 for _ in handle)
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def _area_scores_line(norm: dict) -> str:
|
| 708 |
+
parts = []
|
| 709 |
+
for module, key in (("通用模型", "general_pct"), ("安全可信", "safety_pct"), ("基本教育", "basic_education_pct"), ("高阶育人", "high_level_pct")):
|
| 710 |
+
value = norm.get(key)
|
| 711 |
+
if isinstance(value, (int, float)):
|
| 712 |
+
parts.append(f"{area_label(module)} **{value:.1f}**")
|
| 713 |
+
return " · ".join(parts)
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
def run_evaluation(
|
| 717 |
+
api_key,
|
| 718 |
+
model_id,
|
| 719 |
+
judge_model_id,
|
| 720 |
+
use_system_one_judge,
|
| 721 |
+
use_multiturn_chat,
|
| 722 |
+
multiturn_model_id,
|
| 723 |
+
modules,
|
| 724 |
+
files,
|
| 725 |
+
dimensions,
|
| 726 |
+
max_samples,
|
| 727 |
+
concurrency,
|
| 728 |
+
prices,
|
| 729 |
+
total_budget,
|
| 730 |
+
budget_target,
|
| 731 |
+
budget_judge,
|
| 732 |
+
budget_student,
|
| 733 |
+
confirmed,
|
| 734 |
+
accept_unpriced,
|
| 735 |
+
resume_previous=True,
|
| 736 |
+
persist_to_leaderboard=True,
|
| 737 |
+
):
|
| 738 |
+
empty_table: list[list[Any]] = []
|
| 739 |
+
if CATALOG is None:
|
| 740 |
+
yield f"The benchmark data did not load: {_load_catalog.error}", empty_table, None
|
| 741 |
+
return
|
| 742 |
+
secret = (api_key or "").strip()
|
| 743 |
+
if not secret:
|
| 744 |
+
yield "Paste your OpenRouter API key first.", empty_table, None
|
| 745 |
+
return
|
| 746 |
+
if not confirmed:
|
| 747 |
+
yield "Tick the box above to confirm the run can be charged to your OpenRouter key.", empty_table, None
|
| 748 |
+
return
|
| 749 |
+
if not str(model_id or "").strip():
|
| 750 |
+
yield "Choose the model you want to test.", empty_table, None
|
| 751 |
+
return
|
| 752 |
+
if not str(judge_model_id or "").strip():
|
| 753 |
+
yield "Choose a judge model under Advanced settings.", empty_table, None
|
| 754 |
+
return
|
| 755 |
+
try:
|
| 756 |
+
sample_cap = _max_samples(max_samples)
|
| 757 |
+
worker_concurrency = int(concurrency)
|
| 758 |
+
except (TypeError, ValueError) as exc:
|
| 759 |
+
yield public_error(exc, secret), empty_table, None
|
| 760 |
+
return
|
| 761 |
+
total_limit = _budget(total_budget)
|
| 762 |
+
role_limits = _role_budgets(budget_target, budget_judge, budget_student)
|
| 763 |
+
|
| 764 |
+
holder: dict[str, object] = {"status": "starting", "done": False, "error": "", "result": None, "run_id": ""}
|
| 765 |
+
model_holder: dict[str, str] = {}
|
| 766 |
+
|
| 767 |
+
def _worker() -> None:
|
| 768 |
+
try:
|
| 769 |
+
with visitor_api_key(secret):
|
| 770 |
+
plan = _make_plan(
|
| 771 |
+
model_id=model_id,
|
| 772 |
+
judge_model_id=judge_model_id,
|
| 773 |
+
use_system_one_judge=use_system_one_judge,
|
| 774 |
+
use_multiturn_chat=use_multiturn_chat,
|
| 775 |
+
multiturn_model_id=multiturn_model_id,
|
| 776 |
+
modules=modules,
|
| 777 |
+
files=files,
|
| 778 |
+
dimensions=dimensions,
|
| 779 |
+
max_samples=sample_cap,
|
| 780 |
+
concurrency=worker_concurrency,
|
| 781 |
+
prices=prices,
|
| 782 |
+
)
|
| 783 |
+
model_holder["model_id"] = plan.protocol.model_config.model_id
|
| 784 |
+
problems = plan.error_messages() + budget_problems(plan, total_limit, role_limits)
|
| 785 |
+
if plan.needs_price_ack and not accept_unpriced:
|
| 786 |
+
problems.append(
|
| 787 |
+
"OpenRouter lists no price for this model. Tick *Run even though this model has no "
|
| 788 |
+
"listed price* under Advanced settings to run it anyway."
|
| 789 |
+
)
|
| 790 |
+
if problems:
|
| 791 |
+
holder["error"] = "**This run can't start yet:**\n" + "\n".join(f"- {p}" for p in problems)
|
| 792 |
+
return
|
| 793 |
+
resumable = LEADERBOARD_STORE.find_resumable_run(config_hash=plan.config_hash) if resume_previous else None
|
| 794 |
+
active_run_id = (
|
| 795 |
+
str(resumable["run_id"])
|
| 796 |
+
if resumable is not None and resumable.get("run_id")
|
| 797 |
+
else make_run_id(plan.config_hash)
|
| 798 |
+
)
|
| 799 |
+
holder["run_id"] = active_run_id
|
| 800 |
+
holder["status"] = "Continuing" if resumable is not None else "Running"
|
| 801 |
+
holder["result"] = execute_run(
|
| 802 |
+
CATALOG,
|
| 803 |
+
plan,
|
| 804 |
+
max_concurrency=worker_concurrency,
|
| 805 |
+
accept_unpriced=bool(accept_unpriced),
|
| 806 |
+
run_id=active_run_id,
|
| 807 |
+
resume_previous=bool(resume_previous),
|
| 808 |
+
leaderboard_store=LEADERBOARD_STORE,
|
| 809 |
+
persist_to_leaderboard=bool(persist_to_leaderboard),
|
| 810 |
+
total_budget_usd=total_limit,
|
| 811 |
+
role_budgets_usd=role_limits,
|
| 812 |
+
)
|
| 813 |
+
except Exception as exc: # noqa: BLE001
|
| 814 |
+
holder["error"] = public_error(exc, secret)
|
| 815 |
+
finally:
|
| 816 |
+
holder["done"] = True
|
| 817 |
+
|
| 818 |
+
thread = threading.Thread(target=_worker, daemon=True)
|
| 819 |
+
thread.start()
|
| 820 |
+
while not holder["done"]:
|
| 821 |
+
model_key = model_holder.get("model_id", "")
|
| 822 |
+
active_id = str(holder.get("run_id") or "")
|
| 823 |
+
judged = failed = 0
|
| 824 |
+
if model_key and active_id:
|
| 825 |
+
judged = _count_lines(judged_progress_path(CATALOG.config.project_root, active_id, model_key))
|
| 826 |
+
failed = _count_lines(failures_progress_path(CATALOG.config.project_root, active_id, model_key))
|
| 827 |
+
spent = _spent_so_far(active_id)
|
| 828 |
+
limit = f" of your {_usd(total_limit)} limit" if total_limit is not None else ""
|
| 829 |
+
yield (
|
| 830 |
+
f"**{holder['status']}…** {judged:,} answers scored"
|
| 831 |
+
+ (f", {failed:,} hit errors" if failed else "")
|
| 832 |
+
+ f". Spent so far: {_usd(spent)}{limit}. Every finished answer is saved as it completes.",
|
| 833 |
+
empty_table,
|
| 834 |
+
None,
|
| 835 |
+
)
|
| 836 |
+
time.sleep(1)
|
| 837 |
+
thread.join(timeout=1)
|
| 838 |
+
|
| 839 |
+
if holder.get("error"):
|
| 840 |
+
yield str(holder["error"]), empty_table, None
|
| 841 |
+
return
|
| 842 |
+
result = holder.get("result") or {}
|
| 843 |
+
summary = result.get("summary") or {}
|
| 844 |
+
norm = result.get("normalized_scores") or {}
|
| 845 |
+
judged_total = int(summary.get("total_judged") or 0)
|
| 846 |
+
failures = int(summary.get("total_failures") or 0)
|
| 847 |
+
status_label = str(result.get("status") or "completed")
|
| 848 |
+
spent_text = _usd(float(result.get("spent_usd") or 0.0))
|
| 849 |
+
|
| 850 |
+
if result.get("stopped_by_budget"):
|
| 851 |
+
heading = "### Paused at your spending limit"
|
| 852 |
+
body = (
|
| 853 |
+
f"{judged_total:,} answers scored for {spent_text}. Everything finished so far is saved. "
|
| 854 |
+
"Raise the limit and press **Run** again to carry on from here."
|
| 855 |
+
)
|
| 856 |
+
elif status_label == "completed":
|
| 857 |
+
heading = "### Finished"
|
| 858 |
+
body = f"{judged_total:,} answers scored for {spent_text}."
|
| 859 |
+
elif status_label == "interrupted":
|
| 860 |
+
heading = "### Stopped early"
|
| 861 |
+
body = (
|
| 862 |
+
f"{judged_total:,} answers scored for {spent_text} before the run stopped"
|
| 863 |
+
+ (f": {result['interrupted_error']}" if result.get("interrupted_error") else ".")
|
| 864 |
+
+ " Press **Run** again to continue."
|
| 865 |
+
)
|
| 866 |
+
else:
|
| 867 |
+
heading = "### Finished with some errors"
|
| 868 |
+
body = (
|
| 869 |
+
f"{judged_total:,} answers scored for {spent_text}; {failures:,} hit errors, usually rate limits "
|
| 870 |
+
"or timeouts. Press **Run** again to retry only those."
|
| 871 |
+
)
|
| 872 |
+
scores = _area_scores_line(norm)
|
| 873 |
+
lines = [heading, "", body]
|
| 874 |
+
if scores:
|
| 875 |
+
lines += ["", f"Scores on the questions that ran: {scores}"]
|
| 876 |
+
if persist_to_leaderboard:
|
| 877 |
+
lines += ["", "Saved to the visitor-run leaderboard."]
|
| 878 |
+
if status_label == "completed" and (result.get("leaderboard_record") or {}).get("is_full_benchmark"):
|
| 879 |
+
lines[-1] += " It now appears on the leaderboard chart; press **Refresh** there to see it."
|
| 880 |
+
else:
|
| 881 |
+
lines += ["", "Not saved to the leaderboard (you opted out)."]
|
| 882 |
+
lines += ["", f"<sub>Run id `{result.get('run_id')}`</sub>"]
|
| 883 |
+
yield "\n".join(lines), result.get("summary_rows") or empty_table, result.get("download_paths")
|
| 884 |
+
|
| 885 |
+
|
| 886 |
+
# ---------------------------------------------------------------------------
|
| 887 |
+
# Leaderboard
|
| 888 |
+
# ---------------------------------------------------------------------------
|
| 889 |
+
|
| 890 |
+
|
| 891 |
+
def _metric(focus: str, single_task: str) -> str:
|
| 892 |
+
return single_task or focus or "overall"
|
| 893 |
+
|
| 894 |
+
|
| 895 |
+
def leaderboard_view(
|
| 896 |
+
focus,
|
| 897 |
+
single_task,
|
| 898 |
+
cost_metric,
|
| 899 |
+
include_visitor_runs,
|
| 900 |
+
max_price,
|
| 901 |
+
max_size,
|
| 902 |
+
include_undisclosed,
|
| 903 |
+
prices,
|
| 904 |
+
):
|
| 905 |
+
runs = LEADERBOARD_STORE.load_runs()
|
| 906 |
+
snapshot = openrouter_snapshot()
|
| 907 |
+
merged, label = price_snapshot()
|
| 908 |
+
if isinstance(prices, dict):
|
| 909 |
+
merged.update(prices)
|
| 910 |
+
|
| 911 |
+
def keep(candidate: dict[str, Any]) -> bool:
|
| 912 |
+
return passes_filters(
|
| 913 |
+
candidate["cost"], _size_of(candidate["slug"], snapshot), max_price, max_size, include_undisclosed
|
| 914 |
+
)
|
| 915 |
+
|
| 916 |
+
fig, rows, md = build_pareto_frontier(
|
| 917 |
+
runs,
|
| 918 |
+
merged,
|
| 919 |
+
score_metric=_metric(focus, single_task),
|
| 920 |
+
cost_metric=cost_metric or "blended_per_1m",
|
| 921 |
+
include_custom_runs=bool(include_visitor_runs),
|
| 922 |
+
keep=keep,
|
| 923 |
+
)
|
| 924 |
+
return fig, rows, f"{md}\n\n<sub>{label}</sub>"
|
| 925 |
+
|
| 926 |
+
|
| 927 |
+
def refresh_leaderboards(
|
| 928 |
+
focus,
|
| 929 |
+
single_task,
|
| 930 |
+
cost_metric,
|
| 931 |
+
include_visitor_runs,
|
| 932 |
+
max_price,
|
| 933 |
+
max_size,
|
| 934 |
+
include_undisclosed,
|
| 935 |
+
primary_sel,
|
| 936 |
+
baseline_sels,
|
| 937 |
+
granularity,
|
| 938 |
+
prices,
|
| 939 |
+
):
|
| 940 |
+
runs = LEADERBOARD_STORE.sync_from_hub()
|
| 941 |
+
fig, rows, md = leaderboard_view(
|
| 942 |
+
focus, single_task, cost_metric, include_visitor_runs, max_price, max_size, include_undisclosed, prices
|
| 943 |
+
)
|
| 944 |
+
primary_choices, baseline_choices = build_comparison_choices(runs)
|
| 945 |
+
keys = [value for _, value in primary_choices]
|
| 946 |
+
resolved_primary = primary_sel if primary_sel in keys else (keys[0] if keys else None)
|
| 947 |
+
resolved_baselines = [b for b in (baseline_sels or []) if b in keys] or [
|
| 948 |
+
key for key in ("official:deepseek-v4-flash", "official:gpt-5.4") if key in keys and key != resolved_primary
|
| 949 |
+
]
|
| 950 |
+
headers, comp_rows, comp_note = build_comparison_table(resolved_primary, resolved_baselines, granularity, runs)
|
| 951 |
+
return (
|
| 952 |
+
fig,
|
| 953 |
+
rows,
|
| 954 |
+
md,
|
| 955 |
+
custom_leaderboard_rows(runs),
|
| 956 |
+
f"{len(runs)} visitor run(s). {LEADERBOARD_STORE.sync_status}",
|
| 957 |
+
gr.update(choices=primary_choices, value=resolved_primary),
|
| 958 |
+
gr.update(choices=baseline_choices, value=resolved_baselines),
|
| 959 |
+
gr.Dataframe(headers=headers, value=comp_rows, wrap=True),
|
| 960 |
+
comp_note,
|
| 961 |
+
)
|
| 962 |
+
|
| 963 |
+
|
| 964 |
+
def update_comparison(primary_sel, baseline_sels, granularity):
|
| 965 |
+
headers, comp_rows, comp_note = build_comparison_table(
|
| 966 |
+
primary_sel, baseline_sels, granularity, LEADERBOARD_STORE.load_runs()
|
| 967 |
+
)
|
| 968 |
+
return gr.Dataframe(headers=headers, value=comp_rows, wrap=True), comp_note
|
| 969 |
+
|
| 970 |
+
|
| 971 |
+
def _about_markdown() -> str:
|
| 972 |
+
counts: dict[str, int] = {}
|
| 973 |
+
if CATALOG is not None:
|
| 974 |
+
for entry in CATALOG.files:
|
| 975 |
+
counts[entry.module] = counts.get(entry.module, 0) + entry.item_count
|
| 976 |
+
rows = "\n".join(
|
| 977 |
+
f"| **{label}** | {counts.get(module, 0):,} | {description} |"
|
| 978 |
+
for module, (label, description) in AREA_LABELS.items()
|
| 979 |
+
)
|
| 980 |
+
return f"""## What ELBench measures
|
| 981 |
+
|
| 982 |
+
ELBench asks a model the kinds of things a teacher, tutor or school assistant has to get right, and scores
|
| 983 |
+
the answers in four areas. A single overall number never tells the whole story, so every result is shown
|
| 984 |
+
area by area too.
|
| 985 |
+
|
| 986 |
+
| Area | Questions | What it covers |
|
| 987 |
+
|:-----|:---------:|:---------------|
|
| 988 |
+
{rows}
|
| 989 |
+
|
| 990 |
+
## How the scores work
|
| 991 |
+
|
| 992 |
+
- Each area is scored from 0 to 100. **Overall** is the average of the four areas.
|
| 993 |
+
- Questions with a single right answer are checked automatically. Open-ended answers (safety guidance,
|
| 994 |
+
teaching, character and thinking skills) are graded by a judge model against a rubric.
|
| 995 |
+
- The published results include a 95% range from 10,000 bootstrap resamples. When two models' ranges
|
| 996 |
+
overlap, the difference between them could disappear on a re-run, and the chart says so.
|
| 997 |
+
- **Best value** models score higher than every model that costs less. Price is the OpenRouter price per
|
| 998 |
+
million tokens, weighting input three times as heavily as output, a common way to compare models.
|
| 999 |
+
|
| 1000 |
+
## Visitor runs
|
| 1001 |
+
|
| 1002 |
+
- Runs use the same questions, prompts, retries and scoring code as the paper.
|
| 1003 |
+
- Models are called through OpenRouter, which may route a request to any provider that serves the model.
|
| 1004 |
+
- The default judge is Qwen3-235B on OpenRouter. The paper used its own hosted Qwen judge, so judged scores
|
| 1005 |
+
from visitor runs can differ slightly from the published ones.
|
| 1006 |
+
- Only runs that answered every question in an area appear on the chart for that area, so partial runs never
|
| 1007 |
+
compete with complete ones.
|
| 1008 |
+
|
| 1009 |
+
## Your key and your data
|
| 1010 |
+
|
| 1011 |
+
- Your OpenRouter key is used only while your run is going and is removed from every file the run writes.
|
| 1012 |
+
It is never saved or shown on the leaderboard.
|
| 1013 |
+
- If you leave *Add my results to the public leaderboard* ticked, the model name, settings, scores and
|
| 1014 |
+
answers are published in a public Hugging Face dataset.
|
| 1015 |
+
|
| 1016 |
+
## Learn more
|
| 1017 |
+
|
| 1018 |
+
- [Paper (arXiv:2608.09548)]({ELBENCH_PAPER_URL}) · [Code]({ELBENCH_GITHUB_URL}) ·
|
| 1019 |
+
[Questions]({ELBENCH_DATASET_URL}) · [Published results and audits]({ELBENCH_RESULTS_URL}) ·
|
| 1020 |
+
[AI for Education leaderboard]({AI_FOR_EDUCATION_LEADERBOARD_URL})
|
| 1021 |
+
"""
|
| 1022 |
+
|
| 1023 |
+
|
| 1024 |
+
def _zerogpu_noop() -> str:
|
| 1025 |
+
"""No-op handler so Hugging Face ZeroGPU passes its startup check without burning GPU quota."""
|
| 1026 |
+
return "ok"
|
| 1027 |
+
|
| 1028 |
+
|
| 1029 |
+
if spaces is not None and hasattr(spaces, "GPU"):
|
| 1030 |
+
_zerogpu_noop = spaces.GPU(duration=1)(_zerogpu_noop)
|
| 1031 |
+
|
| 1032 |
+
|
| 1033 |
+
def _filter_controls():
|
| 1034 |
+
with gr.Row():
|
| 1035 |
+
max_price = gr.Slider(
|
| 1036 |
+
label="Highest price",
|
| 1037 |
+
minimum=0.1,
|
| 1038 |
+
maximum=PRICE_FILTER_MAX,
|
| 1039 |
+
step=0.1,
|
| 1040 |
+
value=PRICE_FILTER_MAX,
|
| 1041 |
+
info="Dollars per million tokens. All the way right shows every price.",
|
| 1042 |
+
)
|
| 1043 |
+
max_size = gr.Slider(
|
| 1044 |
+
label="Largest model size",
|
| 1045 |
+
minimum=1,
|
| 1046 |
+
maximum=SIZE_FILTER_MAX,
|
| 1047 |
+
step=1,
|
| 1048 |
+
value=SIZE_FILTER_MAX,
|
| 1049 |
+
info="Billions of parameters. All the way right shows every size.",
|
| 1050 |
+
)
|
| 1051 |
+
include_undisclosed = gr.Checkbox(
|
| 1052 |
+
label="Include models that don't publish their size",
|
| 1053 |
+
value=True,
|
| 1054 |
+
info="Most closed models, such as GPT, Claude and Gemini, don't.",
|
| 1055 |
+
)
|
| 1056 |
+
return max_price, max_size, include_undisclosed
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
+
def build_demo() -> gr.Blocks:
|
| 1060 |
+
if CATALOG is None:
|
| 1061 |
+
with gr.Blocks(title="ELBench-live") as demo:
|
| 1062 |
+
gr.Markdown(f"ELBench-live data did not load.\n\n```\n{_load_catalog.error}\n```")
|
| 1063 |
+
with gr.Row(visible=False):
|
| 1064 |
+
_noop_btn = gr.Button("noop")
|
| 1065 |
+
_noop_out = gr.Textbox()
|
| 1066 |
+
_noop_btn.click(_zerogpu_noop, [], [_noop_out], api_name="healthz")
|
| 1067 |
+
return demo
|
| 1068 |
+
|
| 1069 |
+
initial_runs = LEADERBOARD_STORE.load_runs()
|
| 1070 |
+
initial_prices, _ = price_snapshot()
|
| 1071 |
+
init_fig, init_rows, init_md = leaderboard_view(
|
| 1072 |
+
"overall", "", "blended_per_1m", True, PRICE_FILTER_MAX, SIZE_FILTER_MAX, True, initial_prices
|
| 1073 |
+
)
|
| 1074 |
+
primary_choices, baseline_choices = build_comparison_choices(initial_runs)
|
| 1075 |
+
default_primary = primary_choices[0][1] if primary_choices else "official:gpt-5.4"
|
| 1076 |
+
default_baselines = [k for k in ("official:deepseek-v4-flash", "official:gpt-5.4") if k != default_primary]
|
| 1077 |
+
granularity_default = "By Source File / Task (Matched Subset)"
|
| 1078 |
+
init_comp_headers, init_comp_rows, init_comp_note = build_comparison_table(
|
| 1079 |
+
default_primary, default_baselines, granularity_default, initial_runs
|
| 1080 |
+
)
|
| 1081 |
+
default_files = CATALOG.default_files()
|
| 1082 |
+
|
| 1083 |
+
with gr.Blocks(title="ELBench-live") as demo:
|
| 1084 |
+
prices = gr.State(initial_prices)
|
| 1085 |
+
|
| 1086 |
+
with gr.Tabs() as tabs:
|
| 1087 |
+
with gr.Tab("Leaderboard", id="leaderboard"):
|
| 1088 |
+
gr.Markdown(
|
| 1089 |
+
"# How well do AI models handle education?\n"
|
| 1090 |
+
"ELBench scores AI models on 2,939 questions in four areas: **teaching**, **safety**, "
|
| 1091 |
+
"**knowledge & reasoning**, and **character & thinking skills**. Each dot is a model: "
|
| 1092 |
+
"higher scores better, further left costs less."
|
| 1093 |
+
)
|
| 1094 |
+
focus = gr.Radio(label="Area", choices=FOCUS_CHOICES, value="overall")
|
| 1095 |
+
chart_max_price, chart_max_size, chart_undisclosed = _filter_controls()
|
| 1096 |
+
chart = gr.Plot(value=init_fig, show_label=False)
|
| 1097 |
+
takeaway = gr.Markdown(value=init_md)
|
| 1098 |
+
with gr.Row():
|
| 1099 |
+
go_test = gr.Button("Test your own model →", variant="primary")
|
| 1100 |
+
refresh = gr.Button("Refresh", variant="secondary")
|
| 1101 |
+
|
| 1102 |
+
with gr.Accordion("More ways to look at the results", open=False):
|
| 1103 |
+
with gr.Row():
|
| 1104 |
+
single_task = gr.Dropdown(
|
| 1105 |
+
label="One task only",
|
| 1106 |
+
choices=TASK_CHOICES,
|
| 1107 |
+
value="",
|
| 1108 |
+
filterable=True,
|
| 1109 |
+
info="Show the scores for a single task instead of a whole area.",
|
| 1110 |
+
)
|
| 1111 |
+
cost_metric = gr.Dropdown(
|
| 1112 |
+
label="Price measure",
|
| 1113 |
+
choices=pareto_cost_metric_choices(),
|
| 1114 |
+
value="blended_per_1m",
|
| 1115 |
+
)
|
| 1116 |
+
include_visitor_runs = gr.Checkbox(label="Show visitor runs", value=True)
|
| 1117 |
+
value_table = gr.Dataframe(
|
| 1118 |
+
headers=["", "Model", "Source", "OpenRouter id", "Input $/1M", "Output $/1M", "Price", "Score"],
|
| 1119 |
+
value=init_rows,
|
| 1120 |
+
wrap=True,
|
| 1121 |
+
show_label=False,
|
| 1122 |
+
)
|
| 1123 |
+
|
| 1124 |
+
with gr.Accordion("Published results (all 2,939 questions)", open=False):
|
| 1125 |
+
gr.Dataframe(
|
| 1126 |
+
headers=[
|
| 1127 |
+
"Rank",
|
| 1128 |
+
"Model",
|
| 1129 |
+
"Overall [95% range]",
|
| 1130 |
+
"Knowledge & reasoning",
|
| 1131 |
+
"Safety",
|
| 1132 |
+
"Teaching",
|
| 1133 |
+
"Character & thinking",
|
| 1134 |
+
"Questions",
|
| 1135 |
+
],
|
| 1136 |
+
value=official_leaderboard_rows(),
|
| 1137 |
+
wrap=True,
|
| 1138 |
+
show_label=False,
|
| 1139 |
+
)
|
| 1140 |
+
official_model_filter = gr.Dropdown(
|
| 1141 |
+
label="Task-by-task scores for",
|
| 1142 |
+
choices=[("All models", "All Official Models")]
|
| 1143 |
+
+ [m["display_name"] for m in load_official_models()],
|
| 1144 |
+
value="GPT-5.4",
|
| 1145 |
+
)
|
| 1146 |
+
official_file_table = gr.Dataframe(
|
| 1147 |
+
headers=["Model", "Area", "Task", "Questions", "Score", "Pass rate (%)", "Mean raw score"],
|
| 1148 |
+
value=official_per_file_rows("GPT-5.4"),
|
| 1149 |
+
wrap=True,
|
| 1150 |
+
show_label=False,
|
| 1151 |
+
)
|
| 1152 |
+
|
| 1153 |
+
with gr.Accordion("Visitor runs", open=False):
|
| 1154 |
+
runs_status = gr.Markdown(f"{len(initial_runs)} visitor run(s). {LEADERBOARD_STORE.sync_status}")
|
| 1155 |
+
runs_table = gr.Dataframe(
|
| 1156 |
+
headers=[
|
| 1157 |
+
"Run id",
|
| 1158 |
+
"Updated (UTC)",
|
| 1159 |
+
"Model",
|
| 1160 |
+
"Judge",
|
| 1161 |
+
"Status",
|
| 1162 |
+
"Scope",
|
| 1163 |
+
"Answers",
|
| 1164 |
+
"Errors",
|
| 1165 |
+
"Overall",
|
| 1166 |
+
"Knowledge & reasoning",
|
| 1167 |
+
"Safety",
|
| 1168 |
+
"Teaching",
|
| 1169 |
+
"Character & thinking",
|
| 1170 |
+
],
|
| 1171 |
+
value=custom_leaderboard_rows(initial_runs),
|
| 1172 |
+
wrap=True,
|
| 1173 |
+
show_label=False,
|
| 1174 |
+
)
|
| 1175 |
+
|
| 1176 |
+
with gr.Accordion("Compare results side by side", open=False):
|
| 1177 |
+
with gr.Row():
|
| 1178 |
+
comp_primary = gr.Dropdown(
|
| 1179 |
+
label="Result", choices=primary_choices, value=default_primary, filterable=True
|
| 1180 |
+
)
|
| 1181 |
+
comp_baselines = gr.Dropdown(
|
| 1182 |
+
label="Compare with",
|
| 1183 |
+
choices=baseline_choices,
|
| 1184 |
+
value=default_baselines,
|
| 1185 |
+
multiselect=True,
|
| 1186 |
+
filterable=True,
|
| 1187 |
+
)
|
| 1188 |
+
comp_granularity = gr.Radio(
|
| 1189 |
+
label="Detail",
|
| 1190 |
+
choices=[
|
| 1191 |
+
("Task by task (only tasks the first result covers)", "By Source File / Task (Matched Subset)"),
|
| 1192 |
+
("By area", "By Module (0–100 Normalized)"),
|
| 1193 |
+
],
|
| 1194 |
+
value=granularity_default,
|
| 1195 |
+
)
|
| 1196 |
+
comp_note = gr.Markdown(value=init_comp_note)
|
| 1197 |
+
comp_table = gr.Dataframe(
|
| 1198 |
+
headers=init_comp_headers, value=init_comp_rows, wrap=True, show_label=False
|
| 1199 |
+
)
|
| 1200 |
+
|
| 1201 |
+
with gr.Tab("Test a model", id="test"):
|
| 1202 |
+
gr.Markdown(
|
| 1203 |
+
"# Test a model on ELBench\n"
|
| 1204 |
+
"Bring an [OpenRouter API key](" + OPENROUTER_KEYS_PAGE + "), pick a model, and see how it "
|
| 1205 |
+
"does. You'll see the expected cost before anything is charged, and the run stops at your "
|
| 1206 |
+
"spending limit."
|
| 1207 |
+
)
|
| 1208 |
+
api_key = gr.Textbox(
|
| 1209 |
+
label="OpenRouter API key",
|
| 1210 |
+
type="password",
|
| 1211 |
+
placeholder="sk-or-...",
|
| 1212 |
+
info="Used only for this run. Never saved or shown.",
|
| 1213 |
+
)
|
| 1214 |
+
model = gr.Dropdown(
|
| 1215 |
+
label="Model to test",
|
| 1216 |
+
choices=[],
|
| 1217 |
+
value=DEFAULT_TARGET_SLUG,
|
| 1218 |
+
allow_custom_value=True,
|
| 1219 |
+
filterable=True,
|
| 1220 |
+
info="Type to search, or paste any OpenRouter model id.",
|
| 1221 |
+
)
|
| 1222 |
+
with gr.Accordion("Narrow the list by price or size", open=False):
|
| 1223 |
+
pick_max_price, pick_max_size, pick_undisclosed = _filter_controls()
|
| 1224 |
+
models_shown = gr.Markdown()
|
| 1225 |
+
preset = gr.Radio(label="How much to run", choices=_preset_choices(), value=PRESET_QUICK)
|
| 1226 |
+
with gr.Row():
|
| 1227 |
+
total_budget = gr.Number(
|
| 1228 |
+
label="Spending limit (USD)",
|
| 1229 |
+
value=DEFAULT_BUDGET_USD,
|
| 1230 |
+
minimum=0,
|
| 1231 |
+
info="The run pauses here and keeps its progress. Leave empty for no limit.",
|
| 1232 |
+
)
|
| 1233 |
+
persist_to_leaderboard = gr.Checkbox(
|
| 1234 |
+
label="Add my results to the public leaderboard",
|
| 1235 |
+
value=True,
|
| 1236 |
+
info="Shares the model name, settings, answers and scores. Never your key.",
|
| 1237 |
+
)
|
| 1238 |
+
|
| 1239 |
+
with gr.Accordion("Advanced settings", open=False):
|
| 1240 |
+
gr.Markdown("**Judge and simulated student**")
|
| 1241 |
+
with gr.Row():
|
| 1242 |
+
judge_model = gr.Dropdown(
|
| 1243 |
+
label="Judge model",
|
| 1244 |
+
choices=[],
|
| 1245 |
+
value=DEFAULT_CHAT_JUDGE_SLUG,
|
| 1246 |
+
allow_custom_value=True,
|
| 1247 |
+
filterable=True,
|
| 1248 |
+
info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's judge.",
|
| 1249 |
+
)
|
| 1250 |
+
use_system_one_judge = gr.Checkbox(
|
| 1251 |
+
label="Use a decision model as the judge",
|
| 1252 |
+
value=False,
|
| 1253 |
+
info="Cheaper verdict-only models (System 1). Not the paper's setup.",
|
| 1254 |
+
)
|
| 1255 |
+
with gr.Row():
|
| 1256 |
+
use_multiturn_chat = gr.Checkbox(
|
| 1257 |
+
label="Use a separate model as the simulated student",
|
| 1258 |
+
value=False,
|
| 1259 |
+
info="The tutoring task needs a student to talk to. By default the judge model plays the student.",
|
| 1260 |
+
)
|
| 1261 |
+
multiturn_model = gr.Dropdown(
|
| 1262 |
+
label="Simulated student",
|
| 1263 |
+
choices=[],
|
| 1264 |
+
value=DEFAULT_MULTITURN_CHAT_SLUG,
|
| 1265 |
+
visible=False,
|
| 1266 |
+
interactive=False,
|
| 1267 |
+
allow_custom_value=True,
|
| 1268 |
+
filterable=True,
|
| 1269 |
+
)
|
| 1270 |
+
gr.Markdown("**Separate spending limits** (optional, in USD; they apply within the overall limit)")
|
| 1271 |
+
with gr.Row():
|
| 1272 |
+
budget_target = gr.Number(label="Model under test", value=None, minimum=0)
|
| 1273 |
+
budget_judge = gr.Number(label="Judge", value=None, minimum=0)
|
| 1274 |
+
budget_student = gr.Number(label="Simulated student", value=None, minimum=0)
|
| 1275 |
+
gr.Markdown("**Questions**")
|
| 1276 |
+
modules = gr.CheckboxGroup(
|
| 1277 |
+
label="Areas",
|
| 1278 |
+
choices=_module_choices(),
|
| 1279 |
+
value=CATALOG.default_modules(),
|
| 1280 |
+
info="Choosing an area selects all of its tasks.",
|
| 1281 |
+
)
|
| 1282 |
+
files = gr.CheckboxGroup(
|
| 1283 |
+
label="Tasks",
|
| 1284 |
+
choices=_file_choices(list(CATALOG.files)),
|
| 1285 |
+
value=[entry.canonical_name for entry in default_files],
|
| 1286 |
+
)
|
| 1287 |
+
dimensions = gr.CheckboxGroup(
|
| 1288 |
+
label="Only these skills (leave empty for all)",
|
| 1289 |
+
choices=_dimension_choices(default_files),
|
| 1290 |
+
value=[],
|
| 1291 |
+
)
|
| 1292 |
+
with gr.Row():
|
| 1293 |
+
max_samples = gr.Number(
|
| 1294 |
+
label="Stop after this many questions",
|
| 1295 |
+
value=QUICK_CHECK_ITEMS,
|
| 1296 |
+
precision=0,
|
| 1297 |
+
info="Leave empty to run every selected question.",
|
| 1298 |
+
)
|
| 1299 |
+
concurrency = gr.Slider(
|
| 1300 |
+
label="Requests at once",
|
| 1301 |
+
minimum=1,
|
| 1302 |
+
maximum=100,
|
| 1303 |
+
step=1,
|
| 1304 |
+
value=8,
|
| 1305 |
+
info="Higher is faster but more likely to hit rate limits.",
|
| 1306 |
+
)
|
| 1307 |
+
with gr.Row():
|
| 1308 |
+
resume_previous = gr.Checkbox(
|
| 1309 |
+
label="Continue an unfinished run with the same settings", value=True
|
| 1310 |
+
)
|
| 1311 |
+
accept_unpriced = gr.Checkbox(
|
| 1312 |
+
label="Run even though this model has no listed price", value=False
|
| 1313 |
+
)
|
| 1314 |
+
|
| 1315 |
+
estimate_button = gr.Button("Estimate cost", variant="secondary")
|
| 1316 |
+
report = gr.Markdown()
|
| 1317 |
+
with gr.Accordion("Technical details", open=False):
|
| 1318 |
+
report_details = gr.Markdown()
|
| 1319 |
+
confirmed = gr.Checkbox(label="Charge my OpenRouter key for this run, up to my spending limit")
|
| 1320 |
+
run_button = gr.Button("Run", variant="primary")
|
| 1321 |
+
status = gr.Markdown()
|
| 1322 |
+
with gr.Accordion("Detailed results", open=False):
|
| 1323 |
+
table = gr.Dataframe(
|
| 1324 |
+
headers=["Group", "Name", "Count", "Mean score", "Pass rate"], wrap=True, show_label=False
|
| 1325 |
+
)
|
| 1326 |
+
download = gr.File(label="Download answers, scores and errors", file_count="multiple")
|
| 1327 |
+
|
| 1328 |
+
with gr.Tab("About", id="about"):
|
| 1329 |
+
gr.Markdown(_about_markdown())
|
| 1330 |
+
|
| 1331 |
+
# Leaderboard events
|
| 1332 |
+
view_controls = [
|
| 1333 |
+
focus,
|
| 1334 |
+
single_task,
|
| 1335 |
+
cost_metric,
|
| 1336 |
+
include_visitor_runs,
|
| 1337 |
+
chart_max_price,
|
| 1338 |
+
chart_max_size,
|
| 1339 |
+
chart_undisclosed,
|
| 1340 |
+
]
|
| 1341 |
+
view_outputs = [chart, value_table, takeaway]
|
| 1342 |
+
for control in view_controls:
|
| 1343 |
+
control.change(leaderboard_view, view_controls + [prices], view_outputs)
|
| 1344 |
+
refresh.click(
|
| 1345 |
+
refresh_leaderboards,
|
| 1346 |
+
view_controls + [comp_primary, comp_baselines, comp_granularity, prices],
|
| 1347 |
+
[chart, value_table, takeaway, runs_table, runs_status, comp_primary, comp_baselines, comp_table, comp_note],
|
| 1348 |
+
)
|
| 1349 |
+
go_test.click(lambda: gr.Tabs(selected="test"), None, tabs)
|
| 1350 |
+
official_model_filter.change(official_per_file_rows, [official_model_filter], [official_file_table])
|
| 1351 |
+
for control in (comp_primary, comp_baselines, comp_granularity):
|
| 1352 |
+
control.change(update_comparison, [comp_primary, comp_baselines, comp_granularity], [comp_table, comp_note])
|
| 1353 |
+
|
| 1354 |
+
# Test-a-model events
|
| 1355 |
+
pick_filters = [pick_max_price, pick_max_size, pick_undisclosed]
|
| 1356 |
+
demo.load(
|
| 1357 |
+
load_model_pickers,
|
| 1358 |
+
[use_system_one_judge] + pick_filters,
|
| 1359 |
+
[model, judge_model, multiturn_model, prices, models_shown],
|
| 1360 |
+
)
|
| 1361 |
+
for control in pick_filters:
|
| 1362 |
+
control.change(filter_target_models, [model] + pick_filters, [model, models_shown])
|
| 1363 |
+
use_system_one_judge.change(on_toggle_system_one, [use_system_one_judge], [judge_model])
|
| 1364 |
+
use_multiturn_chat.change(on_toggle_multiturn, [use_multiturn_chat], [multiturn_model])
|
| 1365 |
+
preset.change(apply_preset, [preset], [modules, files, dimensions, max_samples])
|
| 1366 |
+
modules.input(on_modules, [modules], [files, dimensions])
|
| 1367 |
+
files.input(on_files, [files], [dimensions])
|
| 1368 |
+
run_inputs = [
|
| 1369 |
+
api_key,
|
| 1370 |
+
model,
|
| 1371 |
+
judge_model,
|
| 1372 |
+
use_system_one_judge,
|
| 1373 |
+
use_multiturn_chat,
|
| 1374 |
+
multiturn_model,
|
| 1375 |
+
modules,
|
| 1376 |
+
files,
|
| 1377 |
+
dimensions,
|
| 1378 |
+
max_samples,
|
| 1379 |
+
concurrency,
|
| 1380 |
+
prices,
|
| 1381 |
+
total_budget,
|
| 1382 |
+
budget_target,
|
| 1383 |
+
budget_judge,
|
| 1384 |
+
budget_student,
|
| 1385 |
+
]
|
| 1386 |
+
estimate_button.click(estimate, run_inputs, [report, report_details])
|
| 1387 |
+
run_button.click(
|
| 1388 |
+
run_evaluation,
|
| 1389 |
+
run_inputs + [confirmed, accept_unpriced, resume_previous, persist_to_leaderboard],
|
| 1390 |
+
[status, table, download],
|
| 1391 |
+
)
|
| 1392 |
+
with gr.Row(visible=False):
|
| 1393 |
+
_noop_btn = gr.Button("noop")
|
| 1394 |
+
_noop_out = gr.Textbox()
|
| 1395 |
+
_noop_btn.click(_zerogpu_noop, [], [_noop_out], api_name="healthz")
|
| 1396 |
+
return demo
|
| 1397 |
+
|
| 1398 |
+
|
| 1399 |
+
demo = build_demo()
|
| 1400 |
+
# Spaces imports this module and calls launch() itself, so the queue has to be set here.
|
| 1401 |
+
demo.queue(default_concurrency_limit=1)
|
| 1402 |
+
|
| 1403 |
+
|
| 1404 |
+
if __name__ == "__main__":
|
| 1405 |
+
demo.launch(theme=gr.themes.Soft())
|
docs/ARCHITECTURE.md
ADDED
|
@@ -0,0 +1,297 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench Architecture
|
| 2 |
+
|
| 3 |
+
This document describes the engineering design of ELBench: the registry, the
|
| 4 |
+
provider abstraction, the execution model, and how to extend the framework.
|
| 5 |
+
For an overview of the benchmark and the leaderboard, see the top-level
|
| 6 |
+
[README](../README.md).
|
| 7 |
+
|
| 8 |
+
ELBench is built as an engineering system, not a one-off script. Four benchmark
|
| 9 |
+
modules are kept in one consistent framework:
|
| 10 |
+
|
| 11 |
+
- `安全可信` / Safety & Trustworthiness
|
| 12 |
+
- `高阶育人` / High-Level Educational Cultivation
|
| 13 |
+
- `基本教育` / Basic Education
|
| 14 |
+
- `通用模型` / General Capability
|
| 15 |
+
|
| 16 |
+
## Principles
|
| 17 |
+
|
| 18 |
+
- Registry-driven loading instead of hardcoded file logic
|
| 19 |
+
- Provider adapters instead of provider-specific branches in the runner
|
| 20 |
+
- Rule-based judging for objective tasks
|
| 21 |
+
- `LLM-as-a-Judge` for subjective and open-ended tasks
|
| 22 |
+
- Stable machine-readable outputs, logs, checkpoints, and summaries
|
| 23 |
+
- Resume support after interruption
|
| 24 |
+
- Configurable concurrency with a default global cap of `100`
|
| 25 |
+
|
| 26 |
+
## What Is Actually Wired Today
|
| 27 |
+
|
| 28 |
+
### Registered benchmark files
|
| 29 |
+
|
| 30 |
+
`configs/file_registry.yaml` currently resolves these files:
|
| 31 |
+
|
| 32 |
+
- `安全拒答.jsonl`
|
| 33 |
+
- `安全引导.jsonl`
|
| 34 |
+
- `安全回答.jsonl`
|
| 35 |
+
- `SATAs.xlsx`
|
| 36 |
+
- `adversarial_prompts.xlsx`
|
| 37 |
+
- `高阶育人-edu.jsonl`
|
| 38 |
+
- `高阶育人-omni.jsonl`
|
| 39 |
+
- `knowledge.yaml`
|
| 40 |
+
- `question.yaml`
|
| 41 |
+
- `cross.yaml`
|
| 42 |
+
- `config_guided_task.yaml`
|
| 43 |
+
- `mmlu_pro_sampled.jsonl`
|
| 44 |
+
- `ceval_sampled.jsonl`
|
| 45 |
+
- `ifeval_sampled.jsonl`
|
| 46 |
+
- `math_500_sampled.jsonl`
|
| 47 |
+
- `aime24_sampled.jsonl`
|
| 48 |
+
- `aime25.jsonl`
|
| 49 |
+
- `aime26.jsonl`
|
| 50 |
+
- `gsm8k_sampled.jsonl`
|
| 51 |
+
|
| 52 |
+
Files are only considered active benchmark inputs after they are registered.
|
| 53 |
+
Data files not present in `configs/file_registry.yaml` are not part of the
|
| 54 |
+
runnable benchmark.
|
| 55 |
+
|
| 56 |
+
### Current judge strategy
|
| 57 |
+
|
| 58 |
+
Objective tasks use rule or reference matching:
|
| 59 |
+
|
| 60 |
+
- `SATAs.xlsx`
|
| 61 |
+
- `高阶育人-omni.jsonl`
|
| 62 |
+
- `mmlu_pro_sampled.jsonl`
|
| 63 |
+
- `ceval_sampled.jsonl`
|
| 64 |
+
- `ifeval_sampled.jsonl`
|
| 65 |
+
- `math_500_sampled.jsonl`
|
| 66 |
+
- `aime24_sampled.jsonl`
|
| 67 |
+
- `aime25.jsonl`
|
| 68 |
+
- `aime26.jsonl`
|
| 69 |
+
- `gsm8k_sampled.jsonl`
|
| 70 |
+
|
| 71 |
+
Subjective or open-ended tasks use `LLM-as-a-Judge`:
|
| 72 |
+
|
| 73 |
+
- `安全拒答.jsonl`
|
| 74 |
+
- `安全引导.jsonl`
|
| 75 |
+
- `安全回答.jsonl`
|
| 76 |
+
- `adversarial_prompts.xlsx`
|
| 77 |
+
- `高阶育人-edu.jsonl`
|
| 78 |
+
|
| 79 |
+
The repository still treats external provider integration as pluggable.
|
| 80 |
+
`mock.default` and `mock.judge` are included so the full pipeline can be
|
| 81 |
+
validated locally without real API access.
|
| 82 |
+
|
| 83 |
+
## Repository Layout
|
| 84 |
+
|
| 85 |
+
```text
|
| 86 |
+
.
|
| 87 |
+
├── configs/
|
| 88 |
+
│ ├── app.yaml
|
| 89 |
+
│ ├── basic_education.yaml
|
| 90 |
+
│ ├── field_mappings.yaml
|
| 91 |
+
│ ├── file_registry.yaml
|
| 92 |
+
│ ├── judges.yaml
|
| 93 |
+
│ ├── models.yaml
|
| 94 |
+
│ ├── module_registry.yaml
|
| 95 |
+
│ └── providers.yaml
|
| 96 |
+
├── data/
|
| 97 |
+
│ └── benchmark_root/
|
| 98 |
+
│ ├── 安全可信/
|
| 99 |
+
│ ├── 高阶育人/
|
| 100 |
+
│ ├── 基本教育/
|
| 101 |
+
│ └── 通用/
|
| 102 |
+
├── docs/
|
| 103 |
+
│ └── TEAM_MAINTENANCE_GUIDE.zh-CN.md
|
| 104 |
+
├── results/
|
| 105 |
+
├── scripts/
|
| 106 |
+
│ └── run_benchmark.py
|
| 107 |
+
├── src/
|
| 108 |
+
│ └── elbench/
|
| 109 |
+
│ ├── basic_education_runtime/
|
| 110 |
+
│ ├── cli.py
|
| 111 |
+
│ ├── config/
|
| 112 |
+
│ ├── execution/
|
| 113 |
+
│ ├── judges/
|
| 114 |
+
│ ├── loaders/
|
| 115 |
+
│ ├── persistence/
|
| 116 |
+
│ ├── providers/
|
| 117 |
+
│ ├── registry/
|
| 118 |
+
│ ├── schemas/
|
| 119 |
+
│ ├── summary/
|
| 120 |
+
│ └── utils/
|
| 121 |
+
└── tests/
|
| 122 |
+
```
|
| 123 |
+
|
| 124 |
+
`scripts/run_benchmark.py` is intentionally a thin entrypoint. The actual
|
| 125 |
+
benchmark logic lives under `src/elbench/`.
|
| 126 |
+
|
| 127 |
+
## Core Architecture
|
| 128 |
+
|
| 129 |
+
### Registry and schema adaptation
|
| 130 |
+
|
| 131 |
+
- `configs/file_registry.yaml` maps files to module, subset, task, loader, and
|
| 132 |
+
canonical name.
|
| 133 |
+
- `configs/field_mappings.yaml` maps raw dataset fields into ELBench's unified
|
| 134 |
+
internal schema.
|
| 135 |
+
- JSONL, XLSX, and basic-education YAML inputs are normalized into the same
|
| 136 |
+
`Sample` structure.
|
| 137 |
+
|
| 138 |
+
### Unified sample/result schema
|
| 139 |
+
|
| 140 |
+
Loaded records are normalized into a stable schema with fields such as:
|
| 141 |
+
|
| 142 |
+
- `sample_id`
|
| 143 |
+
- `source_file`
|
| 144 |
+
- `module`
|
| 145 |
+
- `subset`
|
| 146 |
+
- `task`
|
| 147 |
+
- `dimension`
|
| 148 |
+
- `prompt`
|
| 149 |
+
- `reference`
|
| 150 |
+
- `metadata`
|
| 151 |
+
|
| 152 |
+
Execution output includes:
|
| 153 |
+
|
| 154 |
+
- `provider_name`
|
| 155 |
+
- `model_id`
|
| 156 |
+
- `model_name`
|
| 157 |
+
- `model_response`
|
| 158 |
+
- `judge_result`
|
| 159 |
+
- `score`
|
| 160 |
+
- `judge_reason`
|
| 161 |
+
- `judge_metadata`
|
| 162 |
+
- `latency_ms`
|
| 163 |
+
- `retry_count`
|
| 164 |
+
- `timestamp`
|
| 165 |
+
|
| 166 |
+
### Provider abstraction
|
| 167 |
+
|
| 168 |
+
- `ModelClient` is the common interface for model execution.
|
| 169 |
+
- Provider-specific API differences are isolated under `src/elbench/providers/`.
|
| 170 |
+
- The main runner does not care whether a model comes from OpenAI, Anthropic,
|
| 171 |
+
Gemini, or another vendor once the adapter exists.
|
| 172 |
+
|
| 173 |
+
### Execution, concurrency, and recovery
|
| 174 |
+
|
| 175 |
+
The runner supports:
|
| 176 |
+
|
| 177 |
+
- global concurrency control
|
| 178 |
+
- provider/model-level concurrency limits
|
| 179 |
+
- retry with exponential backoff
|
| 180 |
+
- failure logging
|
| 181 |
+
- checkpoint-based resume
|
| 182 |
+
- raw-response persistence
|
| 183 |
+
- judged-result persistence
|
| 184 |
+
- summary generation
|
| 185 |
+
|
| 186 |
+
Default global max concurrency is `100` in `configs/app.yaml`.
|
| 187 |
+
|
| 188 |
+
### Basic Education runtime
|
| 189 |
+
|
| 190 |
+
`基本教育` differs from the other modules only in execution style, not in data
|
| 191 |
+
governance:
|
| 192 |
+
|
| 193 |
+
- benchmark data stays under `data/benchmark_root/基本教育/`
|
| 194 |
+
- file discovery still goes through `configs/file_registry.yaml`
|
| 195 |
+
- runtime execution is handled by `src/elbench/execution/basic_education.py`
|
| 196 |
+
- the internal runtime implementation lives under
|
| 197 |
+
`src/elbench/basic_education_runtime/`
|
| 198 |
+
|
| 199 |
+
This internal runtime was adapted from the ELMES project, but it is now packaged
|
| 200 |
+
and maintained as part of ELBench itself.
|
| 201 |
+
|
| 202 |
+
## Output Layout
|
| 203 |
+
|
| 204 |
+
```text
|
| 205 |
+
outputs/
|
| 206 |
+
├── raw_responses/
|
| 207 |
+
├── judged_results/
|
| 208 |
+
├── summaries/
|
| 209 |
+
└── logs/
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
These directories are runtime artifacts and are intentionally ignored by Git.
|
| 213 |
+
Curated leaderboards and audit summaries are committed under `results/`.
|
| 214 |
+
|
| 215 |
+
## Configuration Surface
|
| 216 |
+
|
| 217 |
+
### `configs/models.yaml`
|
| 218 |
+
|
| 219 |
+
Concrete evaluated-model and judge-model instances, including timeout, retry,
|
| 220 |
+
rate limits, and provider kwargs.
|
| 221 |
+
|
| 222 |
+
### `configs/providers.yaml`
|
| 223 |
+
|
| 224 |
+
Provider adapter registry plus default provider capabilities. Current built-in
|
| 225 |
+
adapters:
|
| 226 |
+
|
| 227 |
+
- `mock`
|
| 228 |
+
- `openai_compatible`
|
| 229 |
+
|
| 230 |
+
### `configs/judges.yaml`
|
| 231 |
+
|
| 232 |
+
Task-to-judge routing:
|
| 233 |
+
|
| 234 |
+
- `rule`
|
| 235 |
+
- `llm`
|
| 236 |
+
|
| 237 |
+
### `configs/basic_education.yaml`
|
| 238 |
+
|
| 239 |
+
Scenario-level runtime settings for the internal basic education runtime. This
|
| 240 |
+
file does not replace the registry; it complements it.
|
| 241 |
+
|
| 242 |
+
## Extending ELBench
|
| 243 |
+
|
| 244 |
+
### Add a new provider
|
| 245 |
+
|
| 246 |
+
1. Implement an adapter under `src/elbench/providers/`.
|
| 247 |
+
2. Register it in `configs/providers.yaml`.
|
| 248 |
+
3. Add concrete models in `configs/models.yaml`.
|
| 249 |
+
|
| 250 |
+
### Add a new benchmark file
|
| 251 |
+
|
| 252 |
+
1. Put the data file under `data/benchmark_root/`.
|
| 253 |
+
2. Register it in `configs/file_registry.yaml`.
|
| 254 |
+
3. Add field mappings in `configs/field_mappings.yaml`.
|
| 255 |
+
4. Route judging in `configs/judges.yaml`.
|
| 256 |
+
5. Add or reuse tests.
|
| 257 |
+
|
| 258 |
+
### Add a future module
|
| 259 |
+
|
| 260 |
+
ELBench already reserves space for further modules. The intended path remains:
|
| 261 |
+
|
| 262 |
+
1. add module registration
|
| 263 |
+
2. add file registration
|
| 264 |
+
3. add field mappings
|
| 265 |
+
4. add judge routing
|
| 266 |
+
5. add task-specific logic only if configuration cannot express the behavior
|
| 267 |
+
|
| 268 |
+
No main-runner rewrite should be required.
|
| 269 |
+
|
| 270 |
+
## Maintenance
|
| 271 |
+
|
| 272 |
+
The canonical team maintenance guide is
|
| 273 |
+
[`docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md`](TEAM_MAINTENANCE_GUIDE.zh-CN.md). That
|
| 274 |
+
document explains what each maintained directory is for, which paths are
|
| 275 |
+
runtime-only, and what should or should not be committed.
|
| 276 |
+
|
| 277 |
+
## Status
|
| 278 |
+
|
| 279 |
+
Implemented:
|
| 280 |
+
|
| 281 |
+
- config system
|
| 282 |
+
- file registry
|
| 283 |
+
- JSONL/XLSX/YAML loaders
|
| 284 |
+
- unified sample/result schema
|
| 285 |
+
- provider abstraction
|
| 286 |
+
- concurrent runner
|
| 287 |
+
- retries and checkpoint resume
|
| 288 |
+
- raw/judged/summary/log outputs
|
| 289 |
+
- rule judges for objective tasks
|
| 290 |
+
- `LLM-as-a-Judge` execution path for subjective tasks
|
| 291 |
+
- built-in basic-education bridge on top of the internal runtime module
|
| 292 |
+
|
| 293 |
+
Still intentionally unfinished:
|
| 294 |
+
|
| 295 |
+
- production-grade adapters for every external vendor
|
| 296 |
+
- finalized subjective judge rubrics and prompts
|
| 297 |
+
- complete future-module integration beyond the current registered files
|
docs/ARCHITECTURE.zh-CN.md
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench 架构说明
|
| 2 |
+
|
| 3 |
+
本文档描述 ELBench 的工程设计:注册表、provider 抽象、执行模型,以及如何扩展框架。基准与排行榜的总览见顶层 [README](../README.zh-CN.md)。
|
| 4 |
+
|
| 5 |
+
ELBench 不是临时脚本,而是按正式工程维护的评测系统。当前仓库在一个统一框架下承载四个模块:
|
| 6 |
+
|
| 7 |
+
- `安全可信` / Safety & Trustworthiness
|
| 8 |
+
- `高阶育人` / High-Level Educational Cultivation
|
| 9 |
+
- `基本教育` / Basic Education
|
| 10 |
+
- `通用模型` / General Capability
|
| 11 |
+
|
| 12 |
+
`基本教育` 已作为 ELBench 的正式模块接入。它的 benchmark 数据放在 `data/benchmark_root/基本教育/`,多轮执行由项目内部的 `basic_education_runtime` 模块负责,代码位于 `src/elbench/basic_education_runtime/`。这个内部运行时来源于 ELMES 项目,但现在已经按 ELBench 的内部模块方式接管和维护。
|
| 13 |
+
|
| 14 |
+
## 设计原则
|
| 15 |
+
|
| 16 |
+
- 评测流程由注册表和配置驱动,不在主流程里写死文件逻辑
|
| 17 |
+
- 模型调用统一走 provider adapter,不让厂商差异污染主 runner
|
| 18 |
+
- 客观题优先规则判分
|
| 19 |
+
- 主观和开放式任务采用 `LLM-as-a-Judge`
|
| 20 |
+
- 稳定的机器可读输出、日志、检查点与汇总
|
| 21 |
+
- 中断后支持续跑
|
| 22 |
+
- 可配置并发,默认全局上限 `100`
|
| 23 |
+
|
| 24 |
+
## 当前实际接入的内容
|
| 25 |
+
|
| 26 |
+
### 已注册的 benchmark 文件
|
| 27 |
+
|
| 28 |
+
`configs/file_registry.yaml` 当前解析以下文件:
|
| 29 |
+
|
| 30 |
+
- `安全拒答.jsonl`
|
| 31 |
+
- `安全引导.jsonl`
|
| 32 |
+
- `安全回答.jsonl`
|
| 33 |
+
- `SATAs.xlsx`
|
| 34 |
+
- `adversarial_prompts.xlsx`
|
| 35 |
+
- `高阶育人-edu.jsonl`
|
| 36 |
+
- `高阶育人-omni.jsonl`
|
| 37 |
+
- `knowledge.yaml`
|
| 38 |
+
- `question.yaml`
|
| 39 |
+
- `cross.yaml`
|
| 40 |
+
- `config_guided_task.yaml`
|
| 41 |
+
- `mmlu_pro_sampled.jsonl`
|
| 42 |
+
- `ceval_sampled.jsonl`
|
| 43 |
+
- `ifeval_sampled.jsonl`
|
| 44 |
+
- `math_500_sampled.jsonl`
|
| 45 |
+
- `aime24_sampled.jsonl`
|
| 46 |
+
- `aime25.jsonl`
|
| 47 |
+
- `aime26.jsonl`
|
| 48 |
+
- `gsm8k_sampled.jsonl`
|
| 49 |
+
|
| 50 |
+
文件只有在注册之后才会被视为有效的 benchmark 输入。没有出现在 `configs/file_registry.yaml` 中的数据文件,不属于可运行基准的一部分。
|
| 51 |
+
|
| 52 |
+
### 当前判分策略
|
| 53 |
+
|
| 54 |
+
客观题用规则或参考答案匹配:
|
| 55 |
+
|
| 56 |
+
- `SATAs.xlsx`
|
| 57 |
+
- `高阶育人-omni.jsonl`
|
| 58 |
+
- `mmlu_pro_sampled.jsonl`
|
| 59 |
+
- `ceval_sampled.jsonl`
|
| 60 |
+
- `ifeval_sampled.jsonl`
|
| 61 |
+
- `math_500_sampled.jsonl`
|
| 62 |
+
- `aime24_sampled.jsonl`
|
| 63 |
+
- `aime25.jsonl`
|
| 64 |
+
- `aime26.jsonl`
|
| 65 |
+
- `gsm8k_sampled.jsonl`
|
| 66 |
+
|
| 67 |
+
主观或开放式任务用 `LLM-as-a-Judge`:
|
| 68 |
+
|
| 69 |
+
- `安全拒答.jsonl`
|
| 70 |
+
- `安全引导.jsonl`
|
| 71 |
+
- `安全回答.jsonl`
|
| 72 |
+
- `adversarial_prompts.xlsx`
|
| 73 |
+
- `高阶育人-edu.jsonl`
|
| 74 |
+
|
| 75 |
+
仓库把外部 provider 接入设计为可插拔。内置 `mock.default` 和 `mock.judge`,可在没有真实 API 的情况下本地验证完整流水线。
|
| 76 |
+
|
| 77 |
+
## 仓库结构
|
| 78 |
+
|
| 79 |
+
```text
|
| 80 |
+
.
|
| 81 |
+
├── configs/
|
| 82 |
+
│ ├── app.yaml
|
| 83 |
+
│ ├── basic_education.yaml
|
| 84 |
+
│ ├── field_mappings.yaml
|
| 85 |
+
│ ├── file_registry.yaml
|
| 86 |
+
│ ├── judges.yaml
|
| 87 |
+
│ ├── models.yaml
|
| 88 |
+
│ ├── module_registry.yaml
|
| 89 |
+
│ └── providers.yaml
|
| 90 |
+
├── data/
|
| 91 |
+
│ └── benchmark_root/
|
| 92 |
+
│ ├── 安全可信/
|
| 93 |
+
│ ├── 高阶育人/
|
| 94 |
+
│ ├── 基本教育/
|
| 95 |
+
│ └── 通用/
|
| 96 |
+
├── docs/
|
| 97 |
+
│ └── TEAM_MAINTENANCE_GUIDE.zh-CN.md
|
| 98 |
+
├── results/
|
| 99 |
+
├── scripts/
|
| 100 |
+
│ └── run_benchmark.py
|
| 101 |
+
├── src/
|
| 102 |
+
│ └── elbench/
|
| 103 |
+
│ ├── basic_education_runtime/
|
| 104 |
+
│ ├── cli.py
|
| 105 |
+
│ ├── config/
|
| 106 |
+
│ ├── execution/
|
| 107 |
+
│ ├── judges/
|
| 108 |
+
│ ├── loaders/
|
| 109 |
+
│ ├── persistence/
|
| 110 |
+
│ ├── providers/
|
| 111 |
+
│ ├── registry/
|
| 112 |
+
│ ├── schemas/
|
| 113 |
+
│ ├── summary/
|
| 114 |
+
│ └── utils/
|
| 115 |
+
└── tests/
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
`scripts/run_benchmark.py` 刻意只是一个薄入口,真正的评测逻辑都在 `src/elbench/` 下。
|
| 119 |
+
|
| 120 |
+
## 核心架构
|
| 121 |
+
|
| 122 |
+
### 注册表与 schema 适配
|
| 123 |
+
|
| 124 |
+
- `configs/file_registry.yaml` 把文件映射到模块、子集、任务、loader 与规范名。
|
| 125 |
+
- `configs/field_mappings.yaml` 把原始数据集字段映射进 ELBench 的统一内部 schema。
|
| 126 |
+
- JSONL、XLSX、基本教育 YAML 输入都被规范化为同一个 `Sample` 结构。
|
| 127 |
+
|
| 128 |
+
### 统一的 sample/result schema
|
| 129 |
+
|
| 130 |
+
加载后的记录被规范化为稳定 schema,字段包括:
|
| 131 |
+
|
| 132 |
+
- `sample_id`
|
| 133 |
+
- `source_file`
|
| 134 |
+
- `module`
|
| 135 |
+
- `subset`
|
| 136 |
+
- `task`
|
| 137 |
+
- `dimension`
|
| 138 |
+
- `prompt`
|
| 139 |
+
- `reference`
|
| 140 |
+
- `metadata`
|
| 141 |
+
|
| 142 |
+
执行输出包括:
|
| 143 |
+
|
| 144 |
+
- `provider_name`
|
| 145 |
+
- `model_id`
|
| 146 |
+
- `model_name`
|
| 147 |
+
- `model_response`
|
| 148 |
+
- `judge_result`
|
| 149 |
+
- `score`
|
| 150 |
+
- `judge_reason`
|
| 151 |
+
- `judge_metadata`
|
| 152 |
+
- `latency_ms`
|
| 153 |
+
- `retry_count`
|
| 154 |
+
- `timestamp`
|
| 155 |
+
|
| 156 |
+
### Provider 抽象
|
| 157 |
+
|
| 158 |
+
- `ModelClient` 是模型执行的统一接口。
|
| 159 |
+
- 厂商 API 差异都隔离在 `src/elbench/providers/` 下。
|
| 160 |
+
- 一旦 adapter 存在,主 runner 不关心模型来自 OpenAI、Anthropic、Gemini 还是其他厂商。
|
| 161 |
+
|
| 162 |
+
### 执行、并发与恢复
|
| 163 |
+
|
| 164 |
+
runner 支持:
|
| 165 |
+
|
| 166 |
+
- 全局并发控制
|
| 167 |
+
- provider/模型级并发上限
|
| 168 |
+
- 指数退避重试
|
| 169 |
+
- 失败日志
|
| 170 |
+
- 基于检查点的续跑
|
| 171 |
+
- 原始响应持久化
|
| 172 |
+
- 判分结果持久化
|
| 173 |
+
- 汇总生成
|
| 174 |
+
|
| 175 |
+
`configs/app.yaml` 中默认全局最大并发为 `100`。
|
| 176 |
+
|
| 177 |
+
### 基本教育运行时
|
| 178 |
+
|
| 179 |
+
`基本教育` 与其他模块的差别只在执行方式,不在数据治理:
|
| 180 |
+
|
| 181 |
+
- benchmark 数据仍放在 `data/benchmark_root/基本教育/`
|
| 182 |
+
- 文件发现仍走 `configs/file_registry.yaml`
|
| 183 |
+
- 运行时执行由 `src/elbench/execution/basic_education.py` 负责
|
| 184 |
+
- 内部运行时实现位于 `src/elbench/basic_education_runtime/`
|
| 185 |
+
|
| 186 |
+
## 输出结构
|
| 187 |
+
|
| 188 |
+
```text
|
| 189 |
+
outputs/
|
| 190 |
+
├── raw_responses/
|
| 191 |
+
├── judged_results/
|
| 192 |
+
├── summaries/
|
| 193 |
+
└── logs/
|
| 194 |
+
```
|
| 195 |
+
|
| 196 |
+
这些目录是运行时产物,刻意被 Git 忽略。整理后的排行榜与审计汇总提交在 `results/` 下。
|
| 197 |
+
|
| 198 |
+
## 配置面
|
| 199 |
+
|
| 200 |
+
### `configs/models.yaml`
|
| 201 |
+
|
| 202 |
+
具体的被测模型与裁判模型实例,含超时、重试、限速与 provider 参数。
|
| 203 |
+
|
| 204 |
+
### `configs/providers.yaml`
|
| 205 |
+
|
| 206 |
+
provider adapter 注册表与默认 provider 能力。当前内置 adapter:
|
| 207 |
+
|
| 208 |
+
- `mock`
|
| 209 |
+
- `openai_compatible`
|
| 210 |
+
|
| 211 |
+
### `configs/judges.yaml`
|
| 212 |
+
|
| 213 |
+
任务到裁判的路由:
|
| 214 |
+
|
| 215 |
+
- `rule`
|
| 216 |
+
- `llm`
|
| 217 |
+
|
| 218 |
+
### `configs/basic_education.yaml`
|
| 219 |
+
|
| 220 |
+
基本教育内部运行时的场景级设置。该文件不替代注册表,而是对其补充。
|
| 221 |
+
|
| 222 |
+
## 扩展 ELBench
|
| 223 |
+
|
| 224 |
+
### 新增 provider
|
| 225 |
+
|
| 226 |
+
1. 在 `src/elbench/providers/` 下实现 adapter。
|
| 227 |
+
2. 在 `configs/providers.yaml` 中注册。
|
| 228 |
+
3. 在 `configs/models.yaml` 中加入具体模型。
|
| 229 |
+
|
| 230 |
+
### 新增 benchmark 文件
|
| 231 |
+
|
| 232 |
+
1. 把数据文件放到 `data/benchmark_root/` 下。
|
| 233 |
+
2. 在 `configs/file_registry.yaml` 中注册。
|
| 234 |
+
3. 在 `configs/field_mappings.yaml` 中加字段映射。
|
| 235 |
+
4. 在 `configs/judges.yaml` 中路由判分。
|
| 236 |
+
5. 新增或复用测试。
|
| 237 |
+
|
| 238 |
+
### 新增未来模块
|
| 239 |
+
|
| 240 |
+
ELBench 已为更多模块预留空间,预期路径仍是:
|
| 241 |
+
|
| 242 |
+
1. 加模块注册
|
| 243 |
+
2. 加文件注册
|
| 244 |
+
3. 加字段映射
|
| 245 |
+
4. 加裁判路由
|
| 246 |
+
5. 仅当配置无法表达行为时才加任务专属逻辑
|
| 247 |
+
|
| 248 |
+
不需要重写主 runner。
|
| 249 |
+
|
| 250 |
+
## 维护
|
| 251 |
+
|
| 252 |
+
权威的团队维护指南是 [`docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md`](TEAM_MAINTENANCE_GUIDE.zh-CN.md),它说明每个维护目录的用途、哪些路径仅运行时、以及哪些该提交哪些不该提交。
|
| 253 |
+
|
| 254 |
+
## 状态
|
| 255 |
+
|
| 256 |
+
已实现:
|
| 257 |
+
|
| 258 |
+
- 配置系统
|
| 259 |
+
- 文件注册表
|
| 260 |
+
- JSONL/XLSX/YAML loader
|
| 261 |
+
- 统一 sample/result schema
|
| 262 |
+
- provider 抽象
|
| 263 |
+
- 并发 runner
|
| 264 |
+
- 重试与检查点续跑
|
| 265 |
+
- 原始/判分/汇总/日志输出
|
| 266 |
+
- 客观题规则裁判
|
| 267 |
+
- 主观题 `LLM-as-a-Judge` 执行路径
|
| 268 |
+
- 基于内部运行时模块的基本教育桥接
|
| 269 |
+
|
| 270 |
+
仍刻意未完成:
|
| 271 |
+
|
| 272 |
+
- 面向每个外部厂商的生产级 adapter
|
| 273 |
+
- 最终化的主观裁判 rubric 与 prompt
|
| 274 |
+
- 超出当前已注册文件的完整未来模块接入
|
docs/DASHBOARD_TARGETS.md
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench-live dashboard targets
|
| 2 |
+
|
| 3 |
+
Working list for the hosted dashboard (`dashboard/app.py`, `src/elbench/dashboard/`).
|
| 4 |
+
Each target records the issue, the evidence, the agreed resolution, and how we
|
| 5 |
+
know it is done. Work them in order; update **Status** as each one lands.
|
| 6 |
+
|
| 7 |
+
## Goal
|
| 8 |
+
|
| 9 |
+
1. Present the published ELBench results clearly.
|
| 10 |
+
2. Say plainly what ELBench measures.
|
| 11 |
+
3. Make it plug-and-play for anyone with an OpenRouter key to test a model and
|
| 12 |
+
join the leaderboard by default, with fine-grained control available:
|
| 13 |
+
item selection, judge and student models, and a budget overall and per
|
| 14 |
+
component model.
|
| 15 |
+
|
| 16 |
+
## Decisions
|
| 17 |
+
|
| 18 |
+
- **D1 Scoring follows the paper.** Module and overall scores are computed the
|
| 19 |
+
way the published README/paper table is, unless there is a documented reason
|
| 20 |
+
to override it. Safety uses the item-weighted mean score, not pass rate
|
| 21 |
+
(`leaderboard_FINAL.json` in the results dataset uses pass rate; the paper
|
| 22 |
+
table does not).
|
| 23 |
+
- **D2 Chart first.** The leaderboard chart is the landing view. Explanation
|
| 24 |
+
lives in an About tab. Advanced controls are collapsed by default.
|
| 25 |
+
- **D3 Participate by default.** Saving a run to the public leaderboard is on
|
| 26 |
+
by default, with an opt-out. Only runs that cover the scored slice in full
|
| 27 |
+
are plotted against the paper models.
|
| 28 |
+
|
| 29 |
+
## Phase 1 · Correctness
|
| 30 |
+
|
| 31 |
+
### T1 Refresh stale official per-file data
|
| 32 |
+
- **Issue:** `official_results_data.json` per-file stats predate the results
|
| 33 |
+
dataset's audit fixes, so recomputing module scores from them does not
|
| 34 |
+
reproduce the published table.
|
| 35 |
+
- **Evidence:** High-Level education file for Claude-Opus-4.8, DeepSeek-V4-Pro,
|
| 36 |
+
DeepSeek-V4-Flash, Doubao-Seed-2.0-Pro, GLM-5.1 holds the pre-fix pass rate
|
| 37 |
+
(e.g. DeepSeek-V4-Flash 63.2; post-fix 92.6 per
|
| 38 |
+
`audit-judge-integrity/highedu_llmjudge_fix_summary.json`). High-Level gaps up
|
| 39 |
+
to 15 points. Basic Education per-file data is stale for GPT-5.4 and
|
| 40 |
+
DeepSeek-V4-Pro (`question.yaml` avg 11.36 on a 5-point scale).
|
| 41 |
+
- **Resolution:** Rebuild per-file stats from the audited files in
|
| 42 |
+
`ZeroLoss-Lab/ELBench-results`; record the source files and date in the JSON.
|
| 43 |
+
- **Done when:** a test recomputes every module and overall score from
|
| 44 |
+
per-file data and matches the published table within 0.1 for all 9 models.
|
| 45 |
+
- **Status:** done 2026-10-26. Safety, Basic Education, High-Level and Overall reproduce
|
| 46 |
+
the paper within 0.1; General within 0.25, explained by GSM8K. Provenance is in the JSON's
|
| 47 |
+
`provenance` block; `module_ci` and `overall_ci` added from `bootstrap_ci.json`.
|
| 48 |
+
Test: `test_official_per_file_data_reproduces_the_published_table`.
|
| 49 |
+
|
| 50 |
+
### T2 One scoring function for paper models and visitor runs
|
| 51 |
+
- **Issue:** `compute_normalized_scores` aggregates differently from the paper.
|
| 52 |
+
- **Evidence:** Safety uses pass rate (paper: mean score; gap up to 2.7 pts).
|
| 53 |
+
Basic Education is item-weighted (paper: simple mean of the 4 scenarios).
|
| 54 |
+
- **Resolution:** General and High-Level: item-weighted pass rate. Safety:
|
| 55 |
+
item-weighted mean score. Basic Education: mean of the 4 scenario scores,
|
| 56 |
+
each normalized to its rubric maximum. Overall: mean of the 4 modules.
|
| 57 |
+
- **Done when:** the T1 test uses this same function.
|
| 58 |
+
- **Status:** done 2026-10-26. `normalize_file_entry` + `module_scores_from_per_file`.
|
| 59 |
+
Visitor runs also skip missing or out-of-range Basic Education scores
|
| 60 |
+
(`basic_education_valid_means`), the rule that reproduces the audited values.
|
| 61 |
+
|
| 62 |
+
### T3 Keep partial runs off the frontier
|
| 63 |
+
- **Issue:** a run's score covers only the items it ran, but is plotted against
|
| 64 |
+
full-benchmark scores.
|
| 65 |
+
- **Evidence:** a 5-item MMLU run is placed on the Overall frontier at 100%.
|
| 66 |
+
- **Resolution:** a run is plotted for a metric only if it ran every item in
|
| 67 |
+
that metric's scope (all files of the module, or the full benchmark for
|
| 68 |
+
Overall; no `max_samples` or dimension filter). Partial runs stay visible in
|
| 69 |
+
the runs table with their scope.
|
| 70 |
+
- **Done when:** test: the 5-item run is excluded from Overall and General and
|
| 71 |
+
appears on the per-file MMLU view only if that file ran in full.
|
| 72 |
+
- **Status:** done 2026-10-26. `run_covers_metric` compares judged counts per file with the
|
| 73 |
+
published counts (which equal the catalog's 2,939). The chart summary says how many runs were
|
| 74 |
+
left out. Runs now also record their dimension filter.
|
| 75 |
+
|
| 76 |
+
### T4 Remove the fixed-footprint "Estimated full benchmark cost"
|
| 77 |
+
- **Issue:** the metric applies 0.73M input / 1.5M output tokens to every
|
| 78 |
+
model and leaves out judge cost; it is a price blend labelled as an estimate.
|
| 79 |
+
- **Evidence:** GPT-5.4: metric $24.3; the app's own estimator $3.49–$285.
|
| 80 |
+
- **Resolution:** drop it. Default cost axis is the 3:1 blended price per 1M
|
| 81 |
+
tokens; input and output prices remain as options. Once real full runs
|
| 82 |
+
record measured cost (T12), offer "measured cost of a full run".
|
| 83 |
+
- **Status:** done 2026-10-26 (measured-cost option waits on T12).
|
| 84 |
+
|
| 85 |
+
### T5 One dated price snapshot
|
| 86 |
+
- **Issue:** first render uses prices stored in the JSON; Refresh uses live
|
| 87 |
+
prices, so points move (DeepSeek-V4-Pro $0.96 → $0.73).
|
| 88 |
+
- **Resolution:** fetch the live catalog once per process (cached, with
|
| 89 |
+
timestamp), fall back to stored prices, and label the chart "Prices from
|
| 90 |
+
OpenRouter, <date>".
|
| 91 |
+
- **Status:** done 2026-10-26. `price_snapshot()` in `dashboard/app.py`, 6-hour cache.
|
| 92 |
+
|
| 93 |
+
### T6 Show uncertainty
|
| 94 |
+
- **Issue:** the top six are within 2 points with overlapping 95% intervals,
|
| 95 |
+
but the chart presents small differences as wins.
|
| 96 |
+
- **Resolution:** error bars from the published intervals; a plain note when
|
| 97 |
+
the leaders are statistically tied.
|
| 98 |
+
- **Status:** done 2026-10-26 for the current chart: error bars and a "statistically tied" line.
|
| 99 |
+
Carry both into the T13 chart.
|
| 100 |
+
|
| 101 |
+
### T7 Hygiene
|
| 102 |
+
- `README.md` declares `sdk_version: 6.30.0`; 6.4.0 is installed and
|
| 103 |
+
`dashboard/README.md` says 6.4.0. Pick one.
|
| 104 |
+
- `.env` holds secrets and is not git-ignored.
|
| 105 |
+
- **Status:** done 2026-10-26. README set to 6.4.0 (the version pinned in `pyproject.toml` and
|
| 106 |
+
installed); `.env` added to `.gitignore`.
|
| 107 |
+
|
| 108 |
+
## Phase 2 · Presentation and participation
|
| 109 |
+
|
| 110 |
+
### T8 Three tabs: Leaderboard · Test a model · About
|
| 111 |
+
- Leaderboard: one-line description, chart, module focus switch, short
|
| 112 |
+
takeaway, call to action to test a model; tables collapsed below.
|
| 113 |
+
- About: what ELBench measures (four modules in plain words), how scoring
|
| 114 |
+
works, links, and the fidelity notes now in the header.
|
| 115 |
+
- **Status:** done 2026-10-26. `gr.Tabs` with ids; "Test your own model →" switches tabs. About is built from
|
| 116 |
+
`AREA_LABELS` and the catalog counts.
|
| 117 |
+
|
| 118 |
+
### T9 Plain language
|
| 119 |
+
- Replace jargon in visible labels ("Endpoint to test", "Judge endpoint",
|
| 120 |
+
"/api/v1/systemone", "preflight", "build_summary", "Matched Subset",
|
| 121 |
+
"Pareto-optimal") and drop meta commentary from the main flow. Technical
|
| 122 |
+
detail stays available in the estimate's details and the About tab.
|
| 123 |
+
- **Status:** done 2026-10-26. Visible labels use area names and plain task names (`area_label`, `plain_file`);
|
| 124 |
+
the technical plan moved to a collapsed "Technical details" accordion.
|
| 125 |
+
|
| 126 |
+
### T10 Plug-and-play run with sensible defaults
|
| 127 |
+
- Key → model → preset (Quick check / Standard / Full) → estimate → run.
|
| 128 |
+
Defaults: official judge, save to leaderboard on. One confirmation.
|
| 129 |
+
- **Status:** done 2026-10-26. Presets Quick check (5 questions, default) / Automatically checked set /
|
| 130 |
+
Full benchmark; $5 default limit; leaderboard opt-in on; key checked with `GET /api/v1/key` (shows credit
|
| 131 |
+
left); model list loaded on page load from the cached OpenRouter snapshot.
|
| 132 |
+
|
| 133 |
+
### T11 Advanced controls, collapsed
|
| 134 |
+
- Judge model and System 1 judging, simulated-student model, files and
|
| 135 |
+
dimensions, item cap, concurrency, resume, leaderboard opt-out.
|
| 136 |
+
- **Status:** done 2026-10-26. "Advanced settings" accordion holds judge / decision-model judging, simulated
|
| 137 |
+
student, per-component limits, areas/tasks/skills, question cap, concurrency, resume, unpriced models.
|
| 138 |
+
|
| 139 |
+
### T12 Budgets: overall and per component model
|
| 140 |
+
- **Components:** model under test, judge, simulated student (multi-turn).
|
| 141 |
+
- **Before the run:** show the estimate per component against its budget;
|
| 142 |
+
block the run when an estimate's low end already exceeds a budget.
|
| 143 |
+
- **During the run:** meter spend per component from the `usage.cost` that
|
| 144 |
+
OpenRouter returns; when a component or the total reaches its cap, stop
|
| 145 |
+
through the runner's existing quota-exhausted path, so the run is saved as
|
| 146 |
+
partial and can be resumed. Overshoot is bounded by requests already in flight.
|
| 147 |
+
- **Done when:** unit test stops a run at a cap and leaves it resumable; a
|
| 148 |
+
live check with a tiny cap stops after the first requests.
|
| 149 |
+
- **Status:** done 2026-10-26. `elbench.utils.spend` ledger shared with the runtime subprocess; checks in
|
| 150 |
+
`OpenAICompatibleClient`, `SystemOneJudge`, and a LangChain `SpendCallback`. The estimate blocks a run when a
|
| 151 |
+
low estimate exceeds a limit. Unit tests: `tests/test_spend_limits.py`, `tests/test_spend_runtime_callback.py`.
|
| 152 |
+
Live check (GPT-6 Luna, GLM 5.3 Flash judge, $0.003 total): total limit paused at 3/5 and resumed to 5/5;
|
| 153 |
+
judge limit paused at 2/3 with spend split by role; key absent from every output file. Overshoot seen:
|
| 154 |
+
limit $0.0004, spent $0.0011 (requests in flight finish); the estimate now says so.
|
| 155 |
+
|
| 156 |
+
## Phase 3 · Filters and chart
|
| 157 |
+
|
| 158 |
+
### T13 Interactive chart
|
| 159 |
+
- Plotly via `gr.Plot`: hover details, log price axis, error bars, legend
|
| 160 |
+
for paper models vs visitor runs.
|
| 161 |
+
- **Status:** done 2026-10-26. `_render_pareto_figure` returns a Plotly figure: hover shows score, 95% range,
|
| 162 |
+
price and verdict; log price axis with plain-dollar ticks; light error bars; published = blue circles,
|
| 163 |
+
visitor runs = orange diamonds; best-value step line. Labels are placed to avoid dots, whiskers, the line
|
| 164 |
+
and each other, and are dropped (hover still names the point) when no spot is free. Fixed a label bug:
|
| 165 |
+
chart title and axis showed metric keys (`overall`, `blended_per_1m`) instead of their names.
|
| 166 |
+
`plotly` added to `requirements.txt` and the `dashboard` extra.
|
| 167 |
+
|
| 168 |
+
### T14 Price and size filters
|
| 169 |
+
- OpenRouter `/api/v1/models` has prices but no parameter count. 179 of 468
|
| 170 |
+
models carry `hugging_face_id`; the Hugging Face API returns
|
| 171 |
+
`safetensors.total` (checked: Qwen3-235B → 235.09B). Closed models have no
|
| 172 |
+
size, so the size filter gets an "include undisclosed" toggle. Filter on
|
| 173 |
+
total size and show active size when the model name gives it (`-A22B`).
|
| 174 |
+
Cache sizes; do not fetch on every page load.
|
| 175 |
+
- Apply the filters to both the chart and the model picker.
|
| 176 |
+
- **Status:** done 2026-10-26. `elbench.dashboard.model_sizes`: sizes from Hugging Face `safetensors.total`
|
| 177 |
+
cached in `model_sizes.json` (152 repos; refresh with `python -m elbench.dashboard.model_sizes`), with the
|
| 178 |
+
size in the model id as fallback and for active size. 176 of 468 OpenRouter models have a known size.
|
| 179 |
+
The same three controls (highest price, largest size, include undisclosed) sit above the chart and in the
|
| 180 |
+
model picker. Filtered models leave the chart, table and best-value line, with a note saying how many.
|
| 181 |
+
|
| 182 |
+
## Testing
|
| 183 |
+
|
| 184 |
+
- Unit tests in `tests/test_dashboard_faithfulness.py` per target.
|
| 185 |
+
- Light live checks against OpenRouter with a key from `.env` (variable
|
| 186 |
+
`OPENROUTER_API_KEY`): model list, one cheap request with `usage.cost`, a
|
| 187 |
+
budget stop. Keep each check under a few cents.
|
docs/RESULT_STORAGE_SPEC.zh-CN.md
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench 结果保存规范
|
| 2 |
+
|
| 3 |
+
## 1. 目标
|
| 4 |
+
|
| 5 |
+
这份规范只解决一件事:让评测结果在仓库里保存得清楚、可追溯、适合提交。
|
| 6 |
+
|
| 7 |
+
- `outputs/` 只放运行时全量产物。
|
| 8 |
+
- `results/` 只放审核后的正式结果包。
|
| 9 |
+
- Git 默认只跟踪 `results/`,不跟踪 `outputs/`。
|
| 10 |
+
|
| 11 |
+
## 2. 目录边界
|
| 12 |
+
|
| 13 |
+
| 路径 | 是否入库 | 用途 | 说明 |
|
| 14 |
+
|---|---|---|---|
|
| 15 |
+
| `outputs/` | 否 | 运行时全量产物 | 大、杂、可删除,默认由 `.gitignore` 忽略 |
|
| 16 |
+
| `results/` | 是 | 正式结果包 | 小、稳定、适合汇报和复查 |
|
| 17 |
+
| `docs/RESULT_STORAGE_SPEC.zh-CN.md` | 是 | 本规范 | 结果保存规则的唯一规范入口 |
|
| 18 |
+
|
| 19 |
+
`outputs/` 当前标准子目录如下:
|
| 20 |
+
|
| 21 |
+
- `outputs/raw_responses/<run_id>/`
|
| 22 |
+
- `outputs/judged_results/<run_id>/`
|
| 23 |
+
- `outputs/summaries/<run_id>/`
|
| 24 |
+
- `outputs/logs/<run_id>/`
|
| 25 |
+
|
| 26 |
+
## 3. 标准流程
|
| 27 |
+
|
| 28 |
+
1. 所有评测先按真实 `run_id` 落到 `outputs/`。
|
| 29 |
+
2. 跑完后先检查 `summary`、`failures`、日志和样本范围,确认这是不是一次正式可引用的 run。
|
| 30 |
+
3. 只有正式结果才允许“升格”到 `results/`。
|
| 31 |
+
4. 升格时只整理最小必要文件,不复制原始响应、checkpoint、重试记录、全量日志。
|
| 32 |
+
5. 在 `results/` 下创建结果包目录,并补齐 `README.md`、`manifest.json`、`summary.json`。
|
| 33 |
+
6. 提交 Git 时只提交 `results/` 下的正式结果包和必要文档。
|
| 34 |
+
|
| 35 |
+
## 4. 命名规范
|
| 36 |
+
|
| 37 |
+
正式结果包目录名统一为:
|
| 38 |
+
|
| 39 |
+
`results/<model_id>-<scope>-<YYYYMMDD>[-revN]/`
|
| 40 |
+
|
| 41 |
+
规则如下:
|
| 42 |
+
|
| 43 |
+
- `model_id`:必须与 `configs/models.yaml` 中的模型 id 一致。
|
| 44 |
+
- `scope`:推荐使用 `standard`、`basic-education`、`full`、`safety`、`highlevel`、`general`、`custom-<slug>`。
|
| 45 |
+
- `YYYYMMDD`:结果定稿日,不强制等于任务启动日。
|
| 46 |
+
- 同一天同一范围如果重新定稿,追加 `-rev2`、`-rev3`。
|
| 47 |
+
- 真实运行时使用的 `run_id` 不要求和结果包目录名一致,但必须写进 `manifest.json`。
|
| 48 |
+
|
| 49 |
+
## 5. 结果包内容规范
|
| 50 |
+
|
| 51 |
+
### 5.1 必备文件
|
| 52 |
+
|
| 53 |
+
每个正式结果包至少包含:
|
| 54 |
+
|
| 55 |
+
- `README.md`
|
| 56 |
+
- `manifest.json`
|
| 57 |
+
- `summary.json`
|
| 58 |
+
|
| 59 |
+
### 5.2 可选文件
|
| 60 |
+
|
| 61 |
+
可按需增加:
|
| 62 |
+
|
| 63 |
+
- `notes.md`
|
| 64 |
+
- `artifacts/`
|
| 65 |
+
|
| 66 |
+
`artifacts/` 只允许放精选的小文件,例如手工整理后的表格、图表、失败样例摘录,不允许把运行时全量产物整体搬进来。
|
| 67 |
+
|
| 68 |
+
### 5.3 `README.md` 最少字段
|
| 69 |
+
|
| 70 |
+
每个结果包内的 `README.md` 至少写清楚:
|
| 71 |
+
|
| 72 |
+
- `run_id`
|
| 73 |
+
- `model_id`
|
| 74 |
+
- `scope`
|
| 75 |
+
- `finished_at`
|
| 76 |
+
- `source_outputs`
|
| 77 |
+
- `total_judged`
|
| 78 |
+
- `total_failures`
|
| 79 |
+
- 已知 caveat 或未完成项
|
| 80 |
+
|
| 81 |
+
### 5.4 `manifest.json` 必备字段
|
| 82 |
+
|
| 83 |
+
每个结果包内的 `manifest.json` 至少包含:
|
| 84 |
+
|
| 85 |
+
- `schema_version`
|
| 86 |
+
- `result_id`
|
| 87 |
+
- `run_id`
|
| 88 |
+
- `model_id`
|
| 89 |
+
- `provider_name`
|
| 90 |
+
- `benchmark_scope`
|
| 91 |
+
- `finished_at`
|
| 92 |
+
- `git_commit`
|
| 93 |
+
- `source_outputs`
|
| 94 |
+
- `included_files`
|
| 95 |
+
- `excluded_artifacts`
|
| 96 |
+
|
| 97 |
+
可选字段:
|
| 98 |
+
|
| 99 |
+
- `base_url`
|
| 100 |
+
- `judge_model_ids`
|
| 101 |
+
- `notes`
|
| 102 |
+
|
| 103 |
+
## 6. 禁止提交的内容
|
| 104 |
+
|
| 105 |
+
以下内容默认禁止进入 `results/`:
|
| 106 |
+
|
| 107 |
+
- `outputs/` 下的任何目录
|
| 108 |
+
- 全量 `raw_responses/`
|
| 109 |
+
- `*.checkpoint.json`
|
| 110 |
+
- `*.retries.jsonl`
|
| 111 |
+
- `*.failures.jsonl`
|
| 112 |
+
- `*.log`
|
| 113 |
+
- 密钥文件、环境变量导出、任何 secret
|
| 114 |
+
- `smoke-*`、`probe-*`、`tmp-*`、`debug-*` 之类的试跑结果
|
| 115 |
+
|
| 116 |
+
`judged_results/*.jsonl` 默认也不提交。只有在它本身就是正式交付物的一部分,而且单文件不超过 `5 MB` 时,才允许作为 `artifacts/` 的受控附件入库,并且必须在该结果包的 `README.md` 里说明原因。
|
| 117 |
+
|
| 118 |
+
## 7. 大小与稳定性要求
|
| 119 |
+
|
| 120 |
+
- 单个结果包目标大小:`<= 2 MB`
|
| 121 |
+
- 没有明确理由时,不得超过 `20 MB`
|
| 122 |
+
- `summary.json` 一经提交,就视为该次结果的 canonical snapshot
|
| 123 |
+
- 如果只修正文案,可以在原目录内修改
|
| 124 |
+
- 如果指标、范围、样本集或统计口径发生变化,必须新建 `-revN` 目录,而不是静默覆盖旧结果
|
| 125 |
+
|
| 126 |
+
## 8. 清理策略
|
| 127 |
+
|
| 128 |
+
- `outputs/` 里的 `smoke-*`、`probe-*`、`tmp-*`、`debug-*` 可随时本地删除。
|
| 129 |
+
- 已经升格到 `results/` 的 run,其 `outputs/` 只作为本地审计缓存,不再作为仓库正式资产。
|
| 130 |
+
- 需要长期保留的全量原始产物,应转存到仓库外归档位置;`manifest.json` 只记录来源路径,不把大文件带入 Git。
|
| 131 |
+
|
| 132 |
+
## 9. 标准目录模板
|
| 133 |
+
|
| 134 |
+
```text
|
| 135 |
+
results/
|
| 136 |
+
├── README.md
|
| 137 |
+
└── innospark-235b-standard-20260424/
|
| 138 |
+
├── README.md
|
| 139 |
+
├── manifest.json
|
| 140 |
+
├── summary.json
|
| 141 |
+
└── artifacts/ # optional
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
## 10. 当前落地约定
|
| 145 |
+
|
| 146 |
+
- `outputs/` 继续保持 Git ignore。
|
| 147 |
+
- 当前首个标准样例是 `results/innospark-235b-standard-20260424/`。
|
| 148 |
+
- 后续所有正式评测结果都按本规范落入 `results/`,不再直接把运行时垃圾留给 Git 视图。
|
docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ELBench 团队维护指南
|
| 2 |
+
|
| 3 |
+
本指南面向 ELBench 团队维护者,目标是把“什么应该保留、什么应该删、什么地方负责什么”说清楚,减少仓库熵,保证多人协作和大模型协作时都不会把项目越改越乱。
|
| 4 |
+
|
| 5 |
+
## 1. 总原则
|
| 6 |
+
|
| 7 |
+
- 配置优先。新增数据、模型、judge 时,优先改注册表和配置,不直接侵入主流程。
|
| 8 |
+
- 边界清晰。加载、执行、判分、持久化、汇总必须分层,不要混写。
|
| 9 |
+
- 输出稳定。`outputs/` 结构、结果字段、日志字段不能随意改。
|
| 10 |
+
- 数据归位。正式 benchmark 数据统一放在 `data/benchmark_root/` 下,不能散落到根目录或外挂目录。
|
| 11 |
+
- 数据集不删。任何 benchmark 数据集文件都不得因为“暂未接入”“暂未注册”“暂时无测试”而删除;未接入的数据集只能保留并标记为待接入。
|
| 12 |
+
- 轻仓库。缓存、临时分析文档、历史脚本、系统垃圾文件不要留在主仓库。
|
| 13 |
+
|
| 14 |
+
一句话:**优先走“数据目录 -> 注册表 -> 字段映射 -> judge 路由 -> 边界层代码 -> 测试验证”这条路径。**
|
| 15 |
+
|
| 16 |
+
## 2. 当前清理与重构结论
|
| 17 |
+
|
| 18 |
+
这次整理后,以下判断作为当前团队共识:
|
| 19 |
+
|
| 20 |
+
| 路径 | 结论 | 原因 |
|
| 21 |
+
|---|---|---|
|
| 22 |
+
| `tests/` | 保留 | 这是正式回归保障,不是临时目录 |
|
| 23 |
+
| `src/elbench/basic_education_runtime/` | 保留 | `基本教育` 多轮桥接的内置运行时 |
|
| 24 |
+
| `third_party/` | 删除 | 基本教育运行时已内收进项目内部,不再作为外挂目录存在 |
|
| 25 |
+
| `run.sh` / `setup.sh` | 删除 | 过时、未被文档和主流程使用 |
|
| 26 |
+
| `docs/*analysis*.md` | 删除 | 历史分析稿,且已过时 |
|
| 27 |
+
| `docs/sample_command.md` | 删除 | 使用了已废弃的模型配置 |
|
| 28 |
+
| `edu_eval_spec_for_codex.md` | 删除 | 早期实现规格,已被当前代码和 README 替代 |
|
| 29 |
+
| `.DS_Store`、`pytest-cache-files-*`、`__pycache__`、`outputs/` | 不提交,按需本地删除 | 运行时或系统垃圾文件 |
|
| 30 |
+
|
| 31 |
+
额外说明:
|
| 32 |
+
|
| 33 |
+
- ELBench 当前只有 `src/elbench/` 是项目源码根。
|
| 34 |
+
- `基本教育` 的内部运行时模块位于 `src/elbench/basic_education_runtime/`。
|
| 35 |
+
- 这个内部运行时来源于 ELMES,但现在已经按 ELBench 自有模块维护,不再以外挂目录的形式存在。
|
| 36 |
+
|
| 37 |
+
## 3. 顶层目录/文件职责表
|
| 38 |
+
|
| 39 |
+
| 路径 | 是否应长期保留 | 作用 | 什么时候改 |
|
| 40 |
+
|---|---|---|---|
|
| 41 |
+
| `README.md` | 保留 | 英文默认首页,对外说明项目能力、结构、安装与运行方法 | 仓库能力、结构、安装方式变化时 |
|
| 42 |
+
| `README.zh-CN.md` | 保留 | 中文说明 | 与英文 README 同步更新时 |
|
| 43 |
+
| `docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md` | 保留 | 团队内部维护规范和目录职责表 | 维护规范变化时 |
|
| 44 |
+
| `pyproject.toml` | 保留 | 包定义、依赖、可选 extras、CLI 入口 | 增加依赖、调整安装方式时 |
|
| 45 |
+
| `.gitignore` | 保留 | 忽略缓存、输出和系统垃圾文件 | 新增运行时垃圾目录时 |
|
| 46 |
+
| `configs/` | 保留 | 全部配置驱动入口 | 新增模块、文件、模型、judge、并发规则时 |
|
| 47 |
+
| `data/benchmark_root/` | 保留 | 正式 benchmark 数据根目录 | 新增、整理 benchmark 数据时 |
|
| 48 |
+
| `docs/` | 保留,但只留活文档 | 团队文档目录,不放一次性分析稿 | 文档新增或重写时 |
|
| 49 |
+
| `scripts/run_benchmark.py` | 保留 | 运行入口薄封装 | 很少改;只改入口行为 |
|
| 50 |
+
| `src/elbench/` | 保留 | ELBench 正式源码 | 根据职责分层修改 |
|
| 51 |
+
| `tests/` | 保留 | 回归测试与 smoke test | 新增/修改功能后同步补测试 |
|
| 52 |
+
| `results/` | 保留 | 正式结果包目录,只放审核后的可提交结果 | 新增正式结果快照时 |
|
| 53 |
+
| `outputs/` | 不入库 | 运行产物目录 | 本地跑实验时自动生成 |
|
| 54 |
+
|
| 55 |
+
正式结果包规范见 `docs/RESULT_STORAGE_SPEC.zh-CN.md`,当前结果索引见 `results/README.md`。
|
| 56 |
+
|
| 57 |
+
## 4. `configs/` 职责表
|
| 58 |
+
|
| 59 |
+
| 路径 | 作用 | 注意事项 |
|
| 60 |
+
|---|---|---|
|
| 61 |
+
| `configs/app.yaml` | 全局默认配置,如 data root、output root、默认并发、默认 retry | 默认最大并发当前为 `100` |
|
| 62 |
+
| `configs/module_registry.yaml` | 模块注册表 | 这里只定义模块,不定义文件 |
|
| 63 |
+
| `configs/file_registry.yaml` | benchmark 文件注册表 | 文件是否“正式接入”以这里为准 |
|
| 64 |
+
| `configs/field_mappings.yaml` | 原始字段到统一 schema 的映射 | 字段变了优先改这里 |
|
| 65 |
+
| `configs/providers.yaml` | provider 注册与默认能力 | 不放具体模型实例 |
|
| 66 |
+
| `configs/models.yaml` | 被测模型和 judge 模型实例配置 | 单模型超时、限流、provider kwargs 在这里配 |
|
| 67 |
+
| `configs/judges.yaml` | task 到 rule / llm judge 的路由 | judge 路由唯一入口 |
|
| 68 |
+
| `configs/basic_education.yaml` | 基本教育桥接执行配置 | 只管执行,不替代 benchmark 数据注册 |
|
| 69 |
+
|
| 70 |
+
## 5. `src/elbench/` 代码职责表
|
| 71 |
+
|
| 72 |
+
| 路径 | 作用 | 什么时候改 |
|
| 73 |
+
|---|---|---|
|
| 74 |
+
| `src/elbench/cli.py` | CLI 参数入口 | 新增命令行参数时 |
|
| 75 |
+
| `src/elbench/config/loader.py` | 配置加载 | 新增配置文件或 schema 时 |
|
| 76 |
+
| `src/elbench/schemas/config.py` | 配置 schema | 配置结构变更时 |
|
| 77 |
+
| `src/elbench/schemas/evaluation.py` | Sample / Result / Judge schema | 改输出契约时,需极慎重 |
|
| 78 |
+
| `src/elbench/registry/file_registry.py` | 文件发现与解析 | 文件定位逻辑变化时 |
|
| 79 |
+
| `src/elbench/loaders/base.py` | loader 抽象基类 | loader 接口变化时 |
|
| 80 |
+
| `src/elbench/loaders/jsonl_loader.py` | JSONL 加载 | JSONL 读取逻辑变更时 |
|
| 81 |
+
| `src/elbench/loaders/xlsx_loader.py` | XLSX 加载 | Excel 读取逻辑变更时 |
|
| 82 |
+
| `src/elbench/loaders/basic_education_yaml_loader.py` | 基本教育 YAML 样本加载 | 基本教育数据模板解析变化时 |
|
| 83 |
+
| `src/elbench/loaders/normalizer.py` | 统一 sample 归一化 | Sample 构造规则变化时 |
|
| 84 |
+
| `src/elbench/loaders/resolvers.py` | 派生字段解析,如 `dimension` | 某些字段需要特殊派生逻辑时 |
|
| 85 |
+
| `src/elbench/providers/base.py` | 模型客户端统一接口 | provider 抽象变化时 |
|
| 86 |
+
| `src/elbench/providers/factory.py` | provider 工厂 | 新增 provider 类时 |
|
| 87 |
+
| `src/elbench/providers/openai_compatible.py` | OpenAI 兼容 provider | 接 OpenAI-compatible 厂商时 |
|
| 88 |
+
| `src/elbench/providers/mock.py` | 本地 mock provider | 本地 smoke 或 judge 回归时 |
|
| 89 |
+
| `src/elbench/execution/runner.py` | 主执行器 | 执行调度变更时,慎改 |
|
| 90 |
+
| `src/elbench/execution/basic_education.py` | 基本教育多轮桥接 | 调整内部运行时调用、结果导入、恢复逻辑时 |
|
| 91 |
+
| `src/elbench/execution/rate_limit.py` | 并发与限流控制 | 限流策略变化时 |
|
| 92 |
+
| `src/elbench/execution/retry.py` | 重试与退避 | 重试策略变化时 |
|
| 93 |
+
| `src/elbench/judges/router.py` | judge 路由 | task 到 judge 映射变化时 |
|
| 94 |
+
| `src/elbench/judges/base.py` | judge 抽象基类 | judge 接口变化时 |
|
| 95 |
+
| `src/elbench/judges/llm_judge.py` | `LLM-as-a-Judge` 执行器 | 主观题 judge 链路升级时 |
|
| 96 |
+
| `src/elbench/judges/llm_prompting.py` | judge prompt 模板 | rubric 和 judge prompt 调整时 |
|
| 97 |
+
| `src/elbench/judges/judge_safety.py` | 安全可信相关判分 | 安全任务判分逻辑变化时 |
|
| 98 |
+
| `src/elbench/judges/judge_teaching_harm.py` | 教学安全客观题判分 | SATAs 等规则判分变化时 |
|
| 99 |
+
| `src/elbench/judges/judge_highlevel.py` | 高阶育人判分 | `highlevel_edu` / `highlevel_omni` 变化时 |
|
| 100 |
+
| `src/elbench/judges/judge_mmlu.py` | MMLU-Pro / C-Eval 判分 | 通用客观题扩展时 |
|
| 101 |
+
| `src/elbench/judges/judge_ifeval.py` | IFEval 判分 | 指令遵循判分变化时 |
|
| 102 |
+
| `src/elbench/judges/judge_math.py` | Math-500 / AIME 判分 | 数学题判分变化时 |
|
| 103 |
+
| `src/elbench/basic_education_runtime/` | 基本教育内部多轮运行时 | 修改运行时模型编排、导出、评估逻辑时 |
|
| 104 |
+
| `src/elbench/persistence/writers.py` | 结果写盘 | 输出结构变化时 |
|
| 105 |
+
| `src/elbench/persistence/checkpoint.py` | checkpoint 持久化 | 断点续跑行为变化时 |
|
| 106 |
+
| `src/elbench/persistence/logging.py` | 日志初始化 | 日志格式或级别调整时 |
|
| 107 |
+
| `src/elbench/summary/aggregator.py` | 汇总统计 | 新增 summary 维度时 |
|
| 108 |
+
| `src/elbench/utils/parsing.py` | 通用解析工具 | 文本/答案解析工具扩展时 |
|
| 109 |
+
|
| 110 |
+
## 6. `tests/` 职责表
|
| 111 |
+
|
| 112 |
+
| 路径 | 作用 | 什么时候必须更新 |
|
| 113 |
+
|---|---|---|
|
| 114 |
+
| `tests/test_registry_and_loaders.py` | 校验注册表是否能解析当前 bench 文件,以及 loader 是否能产出统一 sample | 新增 benchmark 文件、调整 loader、改 field mapping 后 |
|
| 115 |
+
| `tests/test_judges.py` | 校验各类客观题和 `LLM-as-a-Judge` 路径的基本行为 | 改 judge、改 prompt 解析、改客观题判分逻辑后 |
|
| 116 |
+
| `tests/test_basic_education_bridge.py` | 校验基本教育 45 题配置、场景任务计数和内部运行时结果导入辅助逻辑 | 改基本教育配置或桥接实现后 |
|
| 117 |
+
|
| 118 |
+
## 7. `src/elbench/basic_education_runtime/` 保留范围
|
| 119 |
+
|
| 120 |
+
这个目录是 `基本教育` 的内置运行时,不是独立仓库。
|
| 121 |
+
|
| 122 |
+
当前应保留:
|
| 123 |
+
|
| 124 |
+
- `src/elbench/basic_education_runtime/` 下的源码
|
| 125 |
+
- `src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf`
|
| 126 |
+
|
| 127 |
+
当前不应再出现:
|
| 128 |
+
|
| 129 |
+
- 外挂式旧 third-party 运行时目录
|
| 130 |
+
- 与 ELBench 无关的上游示例、CI、锁文件
|
| 131 |
+
- 单独的外部项目说明口径
|
| 132 |
+
|
| 133 |
+
## 8. 新增评测数据的标准流程
|
| 134 |
+
|
| 135 |
+
在任何“新增/待接入/重构”场景下,都适用一条硬规则:
|
| 136 |
+
|
| 137 |
+
- 不要删除任何 benchmark 数据集文件。未接入的数据集要保留在 `data/benchmark_root/` 下,并通过文档或注册状态明确标记为“待接入”。
|
| 138 |
+
|
| 139 |
+
### 场景 A:新增文件,但仍属于现有模块
|
| 140 |
+
|
| 141 |
+
1. 把数据放到 `data/benchmark_root/` 下正确的模块目录。
|
| 142 |
+
2. 在 `configs/file_registry.yaml` 注册文件。
|
| 143 |
+
3. 在 `configs/field_mappings.yaml` 增加字段映射。
|
| 144 |
+
4. 在 `configs/judges.yaml` 增加或调整 judge 路由。
|
| 145 |
+
5. 如需派生字段逻辑,在 `src/elbench/loaders/resolvers.py` 扩展。
|
| 146 |
+
6. 客观题优先规则判分,主��题再走 `LLM-as-a-Judge`。
|
| 147 |
+
7. 补至少一条测试。
|
| 148 |
+
8. 跑 `inspect` 和单测。
|
| 149 |
+
|
| 150 |
+
### 场景 B:新增一个全新模块
|
| 151 |
+
|
| 152 |
+
1. 在 `configs/module_registry.yaml` 注册模块。
|
| 153 |
+
2. 在 `data/benchmark_root/` 下建立正式目录。
|
| 154 |
+
3. 为该模块每个文件补 `file_registry.yaml`。
|
| 155 |
+
4. 为该模块每个文件补 `field_mappings.yaml`。
|
| 156 |
+
5. 为该模块 task 补 `judges.yaml`。
|
| 157 |
+
6. 只有配置表达不了时,才新增 loader、judge 或执行逻辑。
|
| 158 |
+
|
| 159 |
+
补充:`基本教育` 不是数据组织例外。它的数据也必须放在 `data/benchmark_root/基本教育/`,只是运行时要通过 `src/elbench/execution/basic_education.py` 调用内部 `basic_education_runtime`。
|
| 160 |
+
|
| 161 |
+
## 9. 什么时候优先改配置,什么时候改代码
|
| 162 |
+
|
| 163 |
+
### 优先改配置
|
| 164 |
+
|
| 165 |
+
- 文件路径变化
|
| 166 |
+
- 文件归属模块或子集变化
|
| 167 |
+
- 原始字段名变化
|
| 168 |
+
- task 从 `rule` 切到 `llm`
|
| 169 |
+
- 模型 timeout / retry / rate limit 变化
|
| 170 |
+
|
| 171 |
+
### 必须改代码
|
| 172 |
+
|
| 173 |
+
- 数据格式不是现有 loader 能覆盖的格式
|
| 174 |
+
- 字段派生逻辑无法由映射表达
|
| 175 |
+
- 新 provider API 形态与现有 adapter 不兼容
|
| 176 |
+
- judge 逻辑本身是新的
|
| 177 |
+
- summary 要新增新的聚合方式
|
| 178 |
+
|
| 179 |
+
### 修改 `runner.py` 的前提
|
| 180 |
+
|
| 181 |
+
只有当配置、adapter、judge、loader 都不能表达需求时,才允许改 `src/elbench/execution/runner.py`。改完必须补测试并跑 smoke run。
|
| 182 |
+
|
| 183 |
+
## 10. judge 维护规范
|
| 184 |
+
|
| 185 |
+
### 客观题
|
| 186 |
+
|
| 187 |
+
客观题默认优先规则判分,适用于:
|
| 188 |
+
|
| 189 |
+
- 单选题
|
| 190 |
+
- 多选题
|
| 191 |
+
- 判断题
|
| 192 |
+
- 有稳定标准答案的结构化任务
|
| 193 |
+
|
| 194 |
+
优先修改位置:
|
| 195 |
+
|
| 196 |
+
- `src/elbench/judges/judge_teaching_harm.py`
|
| 197 |
+
- `src/elbench/judges/judge_highlevel.py`
|
| 198 |
+
- `src/elbench/judges/judge_mmlu.py`
|
| 199 |
+
- `src/elbench/judges/judge_ifeval.py`
|
| 200 |
+
- `src/elbench/judges/judge_math.py`
|
| 201 |
+
- `src/elbench/utils/parsing.py`
|
| 202 |
+
|
| 203 |
+
### 主观题
|
| 204 |
+
|
| 205 |
+
主观题优先 `LLM-as-a-Judge`,且 judge model 必须和被测模型解耦。
|
| 206 |
+
|
| 207 |
+
优先修改位置:
|
| 208 |
+
|
| 209 |
+
- `configs/judges.yaml`
|
| 210 |
+
- `src/elbench/judges/llm_prompting.py`
|
| 211 |
+
- `src/elbench/judges/llm_judge.py`
|
| 212 |
+
|
| 213 |
+
禁止:
|
| 214 |
+
|
| 215 |
+
- 让被测模型给自己打分
|
| 216 |
+
- 把 judge prompt 写死在 `runner.py`
|
| 217 |
+
- 把 rubric 散落到无关文件
|
| 218 |
+
|
| 219 |
+
## 11. 模型与 provider 接入规范
|
| 220 |
+
|
| 221 |
+
### 新增模型
|
| 222 |
+
|
| 223 |
+
1. 先确认现有 adapter 是否已支持。
|
| 224 |
+
2. 只改 `configs/models.yaml`。
|
| 225 |
+
3. 如需新的限流或 timeout,也只在该模型配置里加。
|
| 226 |
+
|
| 227 |
+
### 新增 provider
|
| 228 |
+
|
| 229 |
+
1. 在 `src/elbench/providers/` 新增 adapter。
|
| 230 |
+
2. 在 `factory.py` 注册。
|
| 231 |
+
3. 在 `configs/providers.yaml` 增加 provider。
|
| 232 |
+
4. 在 `configs/models.yaml` 增加具体模型实例。
|
| 233 |
+
|
| 234 |
+
要求:
|
| 235 |
+
|
| 236 |
+
- provider 差异必须封装在 adapter 内
|
| 237 |
+
- 不允许在 runner 或 judge 中写 provider-specific 参数名
|
| 238 |
+
|
| 239 |
+
## 12. 与大模型协作维护的要求
|
| 240 |
+
|
| 241 |
+
每次让大模型改 ELBench 时,至少要明确告诉它:
|
| 242 |
+
|
| 243 |
+
- 项目名是 `ELBench`
|
| 244 |
+
- 这是长期维护 benchmark 框架,不是 demo
|
| 245 |
+
- 正式数据根目录是 `data/benchmark_root/`
|
| 246 |
+
- 当前 judge 策略是“客观题 rule,主观题 llm judge”
|
| 247 |
+
- 先看 `README.md`、本指南和相关配置,再改代码
|
| 248 |
+
|
| 249 |
+
并要求它:
|
| 250 |
+
|
| 251 |
+
1. 优先改配置
|
| 252 |
+
2. 新增数据时同步改 `file_registry`、`field_mappings`、`judges`
|
| 253 |
+
3. 改 judge 时补测试
|
| 254 |
+
4. 提交前跑最小验证命令
|
| 255 |
+
|
| 256 |
+
## 13. 提交前检查清单
|
| 257 |
+
|
| 258 |
+
- [ ] 新数据是否放在 `data/benchmark_root/` 正确目录
|
| 259 |
+
- [ ] `configs/file_registry.yaml` 是否已同步更新
|
| 260 |
+
- [ ] `configs/field_mappings.yaml` 是否已同步更新
|
| 261 |
+
- [ ] `configs/judges.yaml` 是否已同步更新
|
| 262 |
+
- [ ] 是否误改输出结构
|
| 263 |
+
- [ ] 是否新增或更新了测试
|
| 264 |
+
- [ ] 是否清掉了 `outputs/`、缓存目录、系统垃圾文件
|
| 265 |
+
- [ ] 是否执行了:
|
| 266 |
+
- `python -m unittest tests.test_basic_education_bridge tests.test_registry_and_loaders tests.test_judges`
|
| 267 |
+
- `python scripts/run_benchmark.py inspect`
|
| 268 |
+
- 至少一次小样本 `run`
|
| 269 |
+
|
| 270 |
+
## 14. 推荐验证命令
|
| 271 |
+
|
| 272 |
+
```bash
|
| 273 |
+
python -m unittest tests.test_basic_education_bridge tests.test_registry_and_loaders tests.test_judges
|
| 274 |
+
python scripts/run_benchmark.py inspect
|
| 275 |
+
python scripts/run_benchmark.py run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
|
| 276 |
+
```
|
| 277 |
+
|
| 278 |
+
## 15. 常见错误
|
| 279 |
+
|
| 280 |
+
### 错误 1:只改代码,不改注册表
|
| 281 |
+
|
| 282 |
+
后果:文件不会被主流程识别,`inspect` 看不到。
|
| 283 |
+
|
| 284 |
+
### 错误 2:字段名变了,直接改 loader
|
| 285 |
+
|
| 286 |
+
正确做法:先看能不能只改 `configs/field_mappings.yaml`。
|
| 287 |
+
|
| 288 |
+
### 错误 3:新增模型时去改 runner
|
| 289 |
+
|
| 290 |
+
正确做法:优先改 `configs/models.yaml`,必要时才新增 adapter。
|
| 291 |
+
|
| 292 |
+
### 错误 4:把一次性分析稿留在 `docs/`
|
| 293 |
+
|
| 294 |
+
正确做法:只保留活文档;临时分析不要长期入库。
|
| 295 |
+
|
| 296 |
+
### 错误 5:把 `outputs/`、缓存或 `.DS_Store` 提交进仓库
|
| 297 |
+
|
| 298 |
+
正确做法:确保它们继续被 `.gitignore` 忽略,并在提交前清理工作区。
|
docs/benchmark_status_2026-05-25.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Benchmark Status 2026-05-25
|
| 2 |
+
|
| 3 |
+
## Current Scope
|
| 4 |
+
|
| 5 |
+
- Cheap relay models currently evaluated without `安全可信`
|
| 6 |
+
- Active modules for relay runs:
|
| 7 |
+
- `通用模型`
|
| 8 |
+
- `高阶育人`
|
| 9 |
+
- `基本教育`
|
| 10 |
+
- Reason:
|
| 11 |
+
- InnoSpark relay may pre-filter safety prompts, which makes `安全可信` unfair for official comparison
|
| 12 |
+
|
| 13 |
+
## Endpoint Notes
|
| 14 |
+
|
| 15 |
+
- Cheap relay models:
|
| 16 |
+
- `https://api.innospark.cn/v1`
|
| 17 |
+
- key env: `INNOSPARK_RELAY_API_KEY`
|
| 18 |
+
- GPT/Claude support endpoint:
|
| 19 |
+
- `http://35.220.164.252:3888/v1`
|
| 20 |
+
- key env: `EXTERNAL_SUPPORT_API_KEY`
|
| 21 |
+
- Judge / evaluate / student endpoint:
|
| 22 |
+
- current `qwen-test` notebook `9501` endpoint became unstable and later returned `404`
|
| 23 |
+
- this must be repaired or replaced before further official judged runs
|
| 24 |
+
|
| 25 |
+
## Officially Usable Results
|
| 26 |
+
|
| 27 |
+
- `gpt-5.4`
|
| 28 |
+
- full official run completed cleanly
|
| 29 |
+
- run id: `full-gpt-5.4-20260519-v2`
|
| 30 |
+
- `gemini-3-flash-preview`
|
| 31 |
+
- `no-safety` relay run completed
|
| 32 |
+
- run id: `fair-gemini-3-flash-preview-nosafety-20260521-v1`
|
| 33 |
+
- `deepseek-v3.2`
|
| 34 |
+
- `no-safety` relay run mostly completed
|
| 35 |
+
- run id: `fair-deepseek-v3.2-nosafety-20260520-v1`
|
| 36 |
+
- still worth a final cleanup check before treating as final
|
| 37 |
+
|
| 38 |
+
## Dirty / Incomplete Runs
|
| 39 |
+
|
| 40 |
+
- `gpt-5.2-pro`
|
| 41 |
+
- run id: `full-gpt-5.2-pro-20260520-v1`
|
| 42 |
+
- interrupted by quota exhaustion
|
| 43 |
+
- `doubao-seed-2-0-pro-260215`
|
| 44 |
+
- run id: `fair-doubao-seed-2-0-pro-260215-nosafety-20260521-v1`
|
| 45 |
+
- interrupted by local checkpoint write `PermissionError`
|
| 46 |
+
- `kimi-k2.6`
|
| 47 |
+
- runs:
|
| 48 |
+
- `fair-kimi-k2.6-nosafety-20260521-v1`
|
| 49 |
+
- `fair-kimi-k2.6-nosafety-20260521-v2`
|
| 50 |
+
- `fair-kimi-k2.6-nosafety-20260521-v3`
|
| 51 |
+
- key findings:
|
| 52 |
+
- relay has hard rate limit around `1 request/second`
|
| 53 |
+
- model-specific throttling was added: `qps: 1`, `max_concurrency: 1`
|
| 54 |
+
- later hit quota exhaustion, then resume attempts became messy
|
| 55 |
+
- checkpoint showed many completed items, but `raw/judged` artifacts were not present
|
| 56 |
+
- conclusion:
|
| 57 |
+
- current Kimi runs are not directly salvageable as official final results
|
| 58 |
+
- `deepseek-r1-250528`
|
| 59 |
+
- official run:
|
| 60 |
+
- `fair-deepseek-r1-250528-nosafety-20260524-v1`
|
| 61 |
+
- probe run:
|
| 62 |
+
- `probe-deepseek-r1-250528-nosafety-20260525-c2t480-v1`
|
| 63 |
+
- key findings:
|
| 64 |
+
- high `ReadTimeout` rate under prior settings
|
| 65 |
+
- later probe failures were caused by broken `qwen-test` judge endpoint `404`
|
| 66 |
+
- conclusion:
|
| 67 |
+
- not usable as final result yet
|
| 68 |
+
|
| 69 |
+
## Remaining Models To Finish Cleanly
|
| 70 |
+
|
| 71 |
+
- `doubao-seed-2-0-pro-260215`
|
| 72 |
+
- `kimi-k2.6`
|
| 73 |
+
- `deepseek-r1-250528`
|
| 74 |
+
- `gemini-3.1-pro-preview`
|
| 75 |
+
- optional later if quota allows:
|
| 76 |
+
- `gpt-5.2-pro`
|
| 77 |
+
|
| 78 |
+
## Important Technical Findings
|
| 79 |
+
|
| 80 |
+
- `kimi-k2.6` needs special low-rate execution
|
| 81 |
+
- `deepseek-r1-250528` likely needs lower concurrency and longer timeout than the default cheap-model settings
|
| 82 |
+
- judge endpoint health must be verified before any further official judged run
|
| 83 |
+
- for any future run, do not trust checkpoint alone
|
| 84 |
+
- future official runs must confirm all of these are being written:
|
| 85 |
+
- `outputs/raw_responses/...`
|
| 86 |
+
- `outputs/judged_results/...`
|
| 87 |
+
- `outputs/logs/...`
|
| 88 |
+
- `outputs/summaries/...`
|
| 89 |
+
|
| 90 |
+
## Next Workstream
|
| 91 |
+
|
| 92 |
+
- Repair or replace the `qwen-test` judge endpoint
|
| 93 |
+
- Run a full-chain smoke test
|
| 94 |
+
- Start organizing completed clean results into:
|
| 95 |
+
- standardized result folders
|
| 96 |
+
- report tables
|
| 97 |
+
- leaderboard
|
pyproject.toml
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=68", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "elbench"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "Config-driven benchmark framework for education LLM evaluation."
|
| 9 |
+
requires-python = ">=3.11"
|
| 10 |
+
dependencies = [
|
| 11 |
+
"httpx>=0.27.0",
|
| 12 |
+
"nltk>=3.8",
|
| 13 |
+
"latex2sympy2_extended>=1.10.1",
|
| 14 |
+
"openpyxl>=3.1.0",
|
| 15 |
+
"pylatexenc>=2.10",
|
| 16 |
+
"pydantic>=2.7.0",
|
| 17 |
+
"PyYAML>=6.0.0",
|
| 18 |
+
"sympy>=1.13.0",
|
| 19 |
+
"word2number>=1.1",
|
| 20 |
+
]
|
| 21 |
+
|
| 22 |
+
[project.optional-dependencies]
|
| 23 |
+
basic-education = [
|
| 24 |
+
"aiosqlite>=0.20.0",
|
| 25 |
+
"click>=8.1.0",
|
| 26 |
+
"grandalf>=0.8",
|
| 27 |
+
"langchain>=0.3.0",
|
| 28 |
+
"langchain-openai>=0.3.0",
|
| 29 |
+
"langgraph>=0.4.0",
|
| 30 |
+
"langgraph-checkpoint-sqlite>=2.0.0",
|
| 31 |
+
"matplotlib>=3.8.0",
|
| 32 |
+
"pandas>=2.2.0",
|
| 33 |
+
"polyfactory>=2.16.0",
|
| 34 |
+
"pyppeteer>=2.0.0",
|
| 35 |
+
"tenacity>=9.0.0",
|
| 36 |
+
"tqdm>=4.66.0",
|
| 37 |
+
"typing-extensions>=4.12.0",
|
| 38 |
+
]
|
| 39 |
+
dev = [
|
| 40 |
+
"pytest>=8.0.0",
|
| 41 |
+
]
|
| 42 |
+
dashboard = [
|
| 43 |
+
"gradio==6.4.0",
|
| 44 |
+
"plotly>=5.20.0",
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
[project.scripts]
|
| 48 |
+
elbench = "elbench.cli:main"
|
| 49 |
+
|
| 50 |
+
[tool.setuptools]
|
| 51 |
+
package-dir = {"" = "src"}
|
| 52 |
+
|
| 53 |
+
[tool.setuptools.packages.find]
|
| 54 |
+
where = ["src"]
|
| 55 |
+
|
| 56 |
+
[tool.setuptools.package-data]
|
| 57 |
+
"elbench.basic_education_runtime.assets.fonts" = ["*.ttf"]
|
| 58 |
+
"elbench.dashboard" = ["*.json"]
|
requirements.txt
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Hugging Face Spaces installs this file before copying /app.
|
| 2 |
+
# Gradio, spaces, and huggingface_hub are preinstalled by the Spaces runtime.
|
| 3 |
+
httpx>=0.27.0
|
| 4 |
+
nltk>=3.8
|
| 5 |
+
latex2sympy2_extended>=1.10.1
|
| 6 |
+
openpyxl>=3.1.0
|
| 7 |
+
pylatexenc>=2.10
|
| 8 |
+
pydantic>=2.7.0
|
| 9 |
+
PyYAML>=6.0.0
|
| 10 |
+
sympy>=1.13.0
|
| 11 |
+
word2number>=1.1
|
| 12 |
+
aiosqlite>=0.20.0
|
| 13 |
+
langchain>=0.3.0,<0.4.0
|
| 14 |
+
langchain-openai>=0.3.0,<0.4.0
|
| 15 |
+
langgraph>=0.4.0,<0.7.0
|
| 16 |
+
langgraph-checkpoint-sqlite>=2.0.0,<3.0.0
|
| 17 |
+
matplotlib>=3.8.0
|
| 18 |
+
pandas>=2.2.0
|
| 19 |
+
plotly>=5.20.0
|
| 20 |
+
polyfactory>=2.16.0
|
| 21 |
+
tenacity>=9.0.0
|
| 22 |
+
tqdm>=4.66.0
|
| 23 |
+
typing-extensions>=4.12.0
|
| 24 |
+
websockets>=12.0,<14.0
|
scripts/build_stage_report.py
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import json
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 11 |
+
OUTPUT_DIR = ROOT / "results" / "stage-report-20260525"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass(frozen=True)
|
| 15 |
+
class RunSpec:
|
| 16 |
+
model_id: str
|
| 17 |
+
run_id: str
|
| 18 |
+
summary_relpath: str
|
| 19 |
+
scope: str
|
| 20 |
+
included_modules: tuple[str, ...]
|
| 21 |
+
excluded_modules: tuple[str, ...]
|
| 22 |
+
notes: tuple[str, ...] = ()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
RUN_SPECS = (
|
| 26 |
+
RunSpec(
|
| 27 |
+
model_id="gpt-5.4",
|
| 28 |
+
run_id="full-gpt-5.4-20260519-v2",
|
| 29 |
+
summary_relpath="outputs/summaries/full-gpt-5.4-20260519-v2/gpt-5.4.summary.json",
|
| 30 |
+
scope="full_scope",
|
| 31 |
+
included_modules=("general", "safety", "highlevel", "basic_education"),
|
| 32 |
+
excluded_modules=(),
|
| 33 |
+
notes=("cleanest currently available full-scope result",),
|
| 34 |
+
),
|
| 35 |
+
RunSpec(
|
| 36 |
+
model_id="deepseek-v3.2",
|
| 37 |
+
run_id="fair-deepseek-v3.2-nosafety-20260520-v1",
|
| 38 |
+
summary_relpath="outputs/summaries/fair-deepseek-v3.2-nosafety-20260520-v1/deepseek-v3.2.summary.json",
|
| 39 |
+
scope="relay_no_safety",
|
| 40 |
+
included_modules=("general", "highlevel", "basic_education"),
|
| 41 |
+
excluded_modules=("safety",),
|
| 42 |
+
notes=("no-safety relay run; final cleanup still recommended.",),
|
| 43 |
+
),
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
GENERAL_TASKS = (
|
| 48 |
+
"mmlu_pro",
|
| 49 |
+
"ceval",
|
| 50 |
+
"ifeval",
|
| 51 |
+
"math_500",
|
| 52 |
+
"aime24",
|
| 53 |
+
"aime25",
|
| 54 |
+
"aime26",
|
| 55 |
+
"gsm8k",
|
| 56 |
+
)
|
| 57 |
+
HIGHLEVEL_TASKS = ("highlevel_edu", "highlevel_omni")
|
| 58 |
+
SAFETY_TASKS = (
|
| 59 |
+
"safety_refusal",
|
| 60 |
+
"safety_guidance",
|
| 61 |
+
"safety_answer",
|
| 62 |
+
"teaching_harm",
|
| 63 |
+
"adversarial_safety",
|
| 64 |
+
)
|
| 65 |
+
BASIC_TASKS = (
|
| 66 |
+
"basic_knowledge_point_explanation",
|
| 67 |
+
"basic_contextualized_question_generation",
|
| 68 |
+
"basic_interdisciplinary_lesson_plan_generation",
|
| 69 |
+
"basic_guided_problem_solving_teaching",
|
| 70 |
+
)
|
| 71 |
+
BASIC_TASK_MAX_SCORE = {
|
| 72 |
+
"basic_knowledge_point_explanation": 10.0,
|
| 73 |
+
"basic_contextualized_question_generation": 5.0,
|
| 74 |
+
"basic_interdisciplinary_lesson_plan_generation": 5.0,
|
| 75 |
+
# Guided task should use pass_rate, not avg_score.
|
| 76 |
+
"basic_guided_problem_solving_teaching": 1.0,
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def load_json(path: Path) -> dict[str, Any]:
|
| 81 |
+
return json.loads(path.read_text(encoding="utf-8"))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def format_pct(value: float | None) -> str:
|
| 85 |
+
if value is None:
|
| 86 |
+
return ""
|
| 87 |
+
return f"{value * 100:.2f}"
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def format_num(value: float | None) -> str:
|
| 91 |
+
if value is None:
|
| 92 |
+
return ""
|
| 93 |
+
return f"{value:.6f}".rstrip("0").rstrip(".")
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def weighted_pass_rate(summary: dict[str, Any], task_names: tuple[str, ...]) -> float | None:
|
| 97 |
+
by_task = summary.get("by_task", {})
|
| 98 |
+
total_count = 0
|
| 99 |
+
weighted_sum = 0.0
|
| 100 |
+
for task in task_names:
|
| 101 |
+
task_data = by_task.get(task)
|
| 102 |
+
if not task_data:
|
| 103 |
+
continue
|
| 104 |
+
count = int(task_data.get("count", 0) or 0)
|
| 105 |
+
pass_rate = task_data.get("pass_rate")
|
| 106 |
+
if count <= 0 or pass_rate is None:
|
| 107 |
+
continue
|
| 108 |
+
total_count += count
|
| 109 |
+
weighted_sum += count * float(pass_rate)
|
| 110 |
+
if total_count == 0:
|
| 111 |
+
return None
|
| 112 |
+
return weighted_sum / total_count
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def weighted_basic_score(summary: dict[str, Any]) -> float | None:
|
| 116 |
+
by_task = summary.get("by_task", {})
|
| 117 |
+
total_count = 0
|
| 118 |
+
weighted_sum = 0.0
|
| 119 |
+
for task in BASIC_TASKS:
|
| 120 |
+
task_data = by_task.get(task)
|
| 121 |
+
if not task_data:
|
| 122 |
+
continue
|
| 123 |
+
count = int(task_data.get("count", 0) or 0)
|
| 124 |
+
if count <= 0:
|
| 125 |
+
continue
|
| 126 |
+
if task == "basic_guided_problem_solving_teaching":
|
| 127 |
+
normalized = float(task_data.get("pass_rate", 0.0) or 0.0)
|
| 128 |
+
else:
|
| 129 |
+
max_score = BASIC_TASK_MAX_SCORE[task]
|
| 130 |
+
normalized = float(task_data.get("avg_score", 0.0) or 0.0) / max_score
|
| 131 |
+
total_count += count
|
| 132 |
+
weighted_sum += count * normalized
|
| 133 |
+
if total_count == 0:
|
| 134 |
+
return None
|
| 135 |
+
return weighted_sum / total_count
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def scope_score(row: dict[str, Any]) -> float | None:
|
| 139 |
+
parts: list[float] = []
|
| 140 |
+
if row["general_score"] is not None:
|
| 141 |
+
parts.append(row["general_score"])
|
| 142 |
+
if row["highlevel_score"] is not None:
|
| 143 |
+
parts.append(row["highlevel_score"])
|
| 144 |
+
if row["basic_score"] is not None:
|
| 145 |
+
parts.append(row["basic_score"])
|
| 146 |
+
if row["scope"] == "full_scope" and row["safety_score"] is not None:
|
| 147 |
+
parts.append(row["safety_score"])
|
| 148 |
+
if not parts:
|
| 149 |
+
return None
|
| 150 |
+
return sum(parts) / len(parts)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def build_rows() -> list[dict[str, Any]]:
|
| 154 |
+
rows: list[dict[str, Any]] = []
|
| 155 |
+
for spec in RUN_SPECS:
|
| 156 |
+
summary_path = ROOT / spec.summary_relpath
|
| 157 |
+
summary = load_json(summary_path)
|
| 158 |
+
row = {
|
| 159 |
+
"model_id": spec.model_id,
|
| 160 |
+
"run_id": spec.run_id,
|
| 161 |
+
"scope": spec.scope,
|
| 162 |
+
"summary_path": spec.summary_relpath.replace("\\", "/"),
|
| 163 |
+
"included_modules": ", ".join(spec.included_modules),
|
| 164 |
+
"excluded_modules": ", ".join(spec.excluded_modules),
|
| 165 |
+
"notes": " | ".join(spec.notes),
|
| 166 |
+
"total_judged": int(summary.get("total_judged", 0) or 0),
|
| 167 |
+
"total_failures": int(summary.get("total_failures", 0) or 0),
|
| 168 |
+
"general_score": weighted_pass_rate(summary, GENERAL_TASKS),
|
| 169 |
+
"highlevel_score": weighted_pass_rate(summary, HIGHLEVEL_TASKS),
|
| 170 |
+
"safety_score": weighted_pass_rate(summary, SAFETY_TASKS),
|
| 171 |
+
"basic_score": weighted_basic_score(summary),
|
| 172 |
+
}
|
| 173 |
+
row["stage_score"] = scope_score(row)
|
| 174 |
+
rows.append(row)
|
| 175 |
+
return rows
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def write_csv(path: Path, rows: list[dict[str, Any]], fieldnames: list[str]) -> None:
|
| 179 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 180 |
+
with path.open("w", encoding="utf-8", newline="") as fh:
|
| 181 |
+
writer = csv.DictWriter(fh, fieldnames=fieldnames, extrasaction="ignore")
|
| 182 |
+
writer.writeheader()
|
| 183 |
+
for row in rows:
|
| 184 |
+
writer.writerow(row)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def make_overall_table(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
| 188 |
+
table = []
|
| 189 |
+
for row in rows:
|
| 190 |
+
table.append(
|
| 191 |
+
{
|
| 192 |
+
"model_id": row["model_id"],
|
| 193 |
+
"scope": row["scope"],
|
| 194 |
+
"run_id": row["run_id"],
|
| 195 |
+
"total_judged": row["total_judged"],
|
| 196 |
+
"total_failures": row["total_failures"],
|
| 197 |
+
"general_score_pct": format_pct(row["general_score"]),
|
| 198 |
+
"highlevel_score_pct": format_pct(row["highlevel_score"]),
|
| 199 |
+
"safety_score_pct": format_pct(row["safety_score"]),
|
| 200 |
+
"basic_score_pct": format_pct(row["basic_score"]),
|
| 201 |
+
"stage_score_pct": format_pct(row["stage_score"]),
|
| 202 |
+
"included_modules": row["included_modules"],
|
| 203 |
+
"excluded_modules": row["excluded_modules"],
|
| 204 |
+
"notes": row["notes"],
|
| 205 |
+
}
|
| 206 |
+
)
|
| 207 |
+
return table
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def make_scope_leaderboard(rows: list[dict[str, Any]], scope: str) -> list[dict[str, Any]]:
|
| 211 |
+
filtered = [row for row in rows if row["scope"] == scope]
|
| 212 |
+
filtered.sort(key=lambda item: (item["stage_score"] is None, -(item["stage_score"] or 0.0), item["model_id"]))
|
| 213 |
+
leaderboard = []
|
| 214 |
+
for index, row in enumerate(filtered, start=1):
|
| 215 |
+
leaderboard.append(
|
| 216 |
+
{
|
| 217 |
+
"rank": index,
|
| 218 |
+
"model_id": row["model_id"],
|
| 219 |
+
"run_id": row["run_id"],
|
| 220 |
+
"total_judged": row["total_judged"],
|
| 221 |
+
"total_failures": row["total_failures"],
|
| 222 |
+
"general_score_pct": format_pct(row["general_score"]),
|
| 223 |
+
"highlevel_score_pct": format_pct(row["highlevel_score"]),
|
| 224 |
+
"safety_score_pct": format_pct(row["safety_score"]),
|
| 225 |
+
"basic_score_pct": format_pct(row["basic_score"]),
|
| 226 |
+
"stage_score_pct": format_pct(row["stage_score"]),
|
| 227 |
+
"notes": row["notes"],
|
| 228 |
+
}
|
| 229 |
+
)
|
| 230 |
+
return leaderboard
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def markdown_table(rows: list[dict[str, Any]], columns: list[tuple[str, str]]) -> str:
|
| 234 |
+
header = "| " + " | ".join(label for _, label in columns) + " |"
|
| 235 |
+
divider = "| " + " | ".join("---" for _ in columns) + " |"
|
| 236 |
+
body = []
|
| 237 |
+
for row in rows:
|
| 238 |
+
body.append("| " + " | ".join(str(row.get(key, "")) for key, _ in columns) + " |")
|
| 239 |
+
return "\n".join([header, divider, *body])
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def write_report(rows: list[dict[str, Any]]) -> None:
|
| 243 |
+
overall = make_overall_table(rows)
|
| 244 |
+
full_scope = make_scope_leaderboard(rows, "full_scope")
|
| 245 |
+
relay_scope = make_scope_leaderboard(rows, "relay_no_safety")
|
| 246 |
+
report_path = OUTPUT_DIR / "README.md"
|
| 247 |
+
|
| 248 |
+
lines = [
|
| 249 |
+
"# Stage Report 2026-05-25",
|
| 250 |
+
"",
|
| 251 |
+
"## Scope",
|
| 252 |
+
"",
|
| 253 |
+
"- This stage report includes only currently usable summaries.",
|
| 254 |
+
"- `gpt-5.4` is a full-scope result.",
|
| 255 |
+
"- `deepseek-v3.2` is a `no-safety` relay result and should only be compared within the same scope.",
|
| 256 |
+
"- Dirty, incomplete, expensive, or incompatible runs such as `gemini-3-flash-preview`, `kimi-k2.6`, `deepseek-r1-250528`, `doubao-seed-2-0-pro-260215`, and `gpt-5.2-pro` are excluded from the leaderboard.",
|
| 257 |
+
"",
|
| 258 |
+
"## Included Runs",
|
| 259 |
+
"",
|
| 260 |
+
markdown_table(
|
| 261 |
+
overall,
|
| 262 |
+
[
|
| 263 |
+
("model_id", "Model"),
|
| 264 |
+
("scope", "Scope"),
|
| 265 |
+
("total_judged", "Judged"),
|
| 266 |
+
("total_failures", "Failures"),
|
| 267 |
+
("general_score_pct", "General %"),
|
| 268 |
+
("highlevel_score_pct", "Highlevel %"),
|
| 269 |
+
("safety_score_pct", "Safety %"),
|
| 270 |
+
("basic_score_pct", "Basic %"),
|
| 271 |
+
("stage_score_pct", "Stage %"),
|
| 272 |
+
],
|
| 273 |
+
),
|
| 274 |
+
"",
|
| 275 |
+
"## Leaderboard: Full Scope",
|
| 276 |
+
"",
|
| 277 |
+
markdown_table(
|
| 278 |
+
full_scope,
|
| 279 |
+
[
|
| 280 |
+
("rank", "Rank"),
|
| 281 |
+
("model_id", "Model"),
|
| 282 |
+
("stage_score_pct", "Stage %"),
|
| 283 |
+
("general_score_pct", "General %"),
|
| 284 |
+
("safety_score_pct", "Safety %"),
|
| 285 |
+
("highlevel_score_pct", "Highlevel %"),
|
| 286 |
+
("basic_score_pct", "Basic %"),
|
| 287 |
+
],
|
| 288 |
+
),
|
| 289 |
+
"",
|
| 290 |
+
"## Leaderboard: Relay No-Safety Scope",
|
| 291 |
+
"",
|
| 292 |
+
markdown_table(
|
| 293 |
+
relay_scope,
|
| 294 |
+
[
|
| 295 |
+
("rank", "Rank"),
|
| 296 |
+
("model_id", "Model"),
|
| 297 |
+
("stage_score_pct", "Stage %"),
|
| 298 |
+
("general_score_pct", "General %"),
|
| 299 |
+
("highlevel_score_pct", "Highlevel %"),
|
| 300 |
+
("basic_score_pct", "Basic %"),
|
| 301 |
+
("total_failures", "Failures"),
|
| 302 |
+
],
|
| 303 |
+
),
|
| 304 |
+
"",
|
| 305 |
+
"## Notes",
|
| 306 |
+
"",
|
| 307 |
+
"- `Stage %` is a normalized stage score built from currently available module-level metrics.",
|
| 308 |
+
"- `Basic %` normalizes basic-education tasks onto a 0-100 scale before aggregation.",
|
| 309 |
+
"- Cross-scope comparison is intentionally avoided: full-scope and no-safety relay runs have different included modules.",
|
| 310 |
+
]
|
| 311 |
+
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def main() -> None:
|
| 315 |
+
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
| 316 |
+
rows = build_rows()
|
| 317 |
+
|
| 318 |
+
overall_rows = make_overall_table(rows)
|
| 319 |
+
full_scope_rows = make_scope_leaderboard(rows, "full_scope")
|
| 320 |
+
relay_scope_rows = make_scope_leaderboard(rows, "relay_no_safety")
|
| 321 |
+
|
| 322 |
+
write_csv(
|
| 323 |
+
OUTPUT_DIR / "summary_table.csv",
|
| 324 |
+
overall_rows,
|
| 325 |
+
[
|
| 326 |
+
"model_id",
|
| 327 |
+
"scope",
|
| 328 |
+
"run_id",
|
| 329 |
+
"total_judged",
|
| 330 |
+
"total_failures",
|
| 331 |
+
"general_score_pct",
|
| 332 |
+
"highlevel_score_pct",
|
| 333 |
+
"safety_score_pct",
|
| 334 |
+
"basic_score_pct",
|
| 335 |
+
"stage_score_pct",
|
| 336 |
+
"included_modules",
|
| 337 |
+
"excluded_modules",
|
| 338 |
+
"notes",
|
| 339 |
+
],
|
| 340 |
+
)
|
| 341 |
+
write_csv(
|
| 342 |
+
OUTPUT_DIR / "leaderboard_full_scope.csv",
|
| 343 |
+
full_scope_rows,
|
| 344 |
+
[
|
| 345 |
+
"rank",
|
| 346 |
+
"model_id",
|
| 347 |
+
"run_id",
|
| 348 |
+
"total_judged",
|
| 349 |
+
"total_failures",
|
| 350 |
+
"general_score_pct",
|
| 351 |
+
"highlevel_score_pct",
|
| 352 |
+
"safety_score_pct",
|
| 353 |
+
"basic_score_pct",
|
| 354 |
+
"stage_score_pct",
|
| 355 |
+
"notes",
|
| 356 |
+
],
|
| 357 |
+
)
|
| 358 |
+
write_csv(
|
| 359 |
+
OUTPUT_DIR / "leaderboard_relay_no_safety.csv",
|
| 360 |
+
relay_scope_rows,
|
| 361 |
+
[
|
| 362 |
+
"rank",
|
| 363 |
+
"model_id",
|
| 364 |
+
"run_id",
|
| 365 |
+
"total_judged",
|
| 366 |
+
"total_failures",
|
| 367 |
+
"general_score_pct",
|
| 368 |
+
"highlevel_score_pct",
|
| 369 |
+
"basic_score_pct",
|
| 370 |
+
"stage_score_pct",
|
| 371 |
+
"notes",
|
| 372 |
+
],
|
| 373 |
+
)
|
| 374 |
+
|
| 375 |
+
manifest = {
|
| 376 |
+
"generated_at": "2026-05-25",
|
| 377 |
+
"result_id": "stage-report-20260525",
|
| 378 |
+
"included_runs": [
|
| 379 |
+
{
|
| 380 |
+
"model_id": row["model_id"],
|
| 381 |
+
"run_id": row["run_id"],
|
| 382 |
+
"scope": row["scope"],
|
| 383 |
+
"summary_path": row["summary_path"],
|
| 384 |
+
"notes": row["notes"],
|
| 385 |
+
}
|
| 386 |
+
for row in rows
|
| 387 |
+
],
|
| 388 |
+
"excluded_runs": [
|
| 389 |
+
{
|
| 390 |
+
"model_id": "kimi-k2.6",
|
| 391 |
+
"reason": "dirty runs; missing raw/judged artifacts",
|
| 392 |
+
},
|
| 393 |
+
{
|
| 394 |
+
"model_id": "deepseek-r1-250528",
|
| 395 |
+
"reason": "dirty runs; timeouts and broken judge endpoint during probe",
|
| 396 |
+
},
|
| 397 |
+
{
|
| 398 |
+
"model_id": "doubao-seed-2-0-pro-260215",
|
| 399 |
+
"reason": "run interrupted by checkpoint write failure",
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"model_id": "gemini-3-flash-preview",
|
| 403 |
+
"reason": "excluded from official scope because the relay repeatedly returns empty content after consuming reasoning tokens",
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"model_id": "gpt-5.2-pro",
|
| 407 |
+
"reason": "expensive GPT pro model intentionally excluded from the current campaign",
|
| 408 |
+
},
|
| 409 |
+
],
|
| 410 |
+
"generated_files": [
|
| 411 |
+
"README.md",
|
| 412 |
+
"summary_table.csv",
|
| 413 |
+
"leaderboard_full_scope.csv",
|
| 414 |
+
"leaderboard_relay_no_safety.csv",
|
| 415 |
+
"manifest.json",
|
| 416 |
+
],
|
| 417 |
+
}
|
| 418 |
+
(OUTPUT_DIR / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 419 |
+
write_report(rows)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
if __name__ == "__main__":
|
| 423 |
+
main()
|
scripts/repair_satas_reference_and_rejudge.py
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import asyncio
|
| 5 |
+
import json
|
| 6 |
+
import shutil
|
| 7 |
+
import sys
|
| 8 |
+
from datetime import datetime
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
SRC = ROOT / "src"
|
| 16 |
+
if str(SRC) not in sys.path:
|
| 17 |
+
sys.path.insert(0, str(SRC))
|
| 18 |
+
|
| 19 |
+
from elbench.config import load_project_config
|
| 20 |
+
from elbench.judges.judge_teaching_harm import TeachingHarmJudge
|
| 21 |
+
from elbench.loaders import LoaderFactory
|
| 22 |
+
from elbench.persistence import OutputPaths
|
| 23 |
+
from elbench.registry import FileRegistry
|
| 24 |
+
from elbench.schemas.evaluation import ModelResponse, Sample
|
| 25 |
+
from elbench.summary import build_summary
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
MODELS_AND_RUNS: dict[str, str] = {
|
| 29 |
+
"claude-opus-4-8": "campaign-official-claude-opus-4-8",
|
| 30 |
+
"deepseek-v4-pro": "campaign-official-deepseek-v4-pro",
|
| 31 |
+
"gemini-3.5-flash": "campaign-official-gemini-3.5-flash-aime65536-highlevel-llmjudge",
|
| 32 |
+
"deepseek-v4-flash": "campaign-official-deepseek-v4-flash",
|
| 33 |
+
"doubao-seed-2.0-pro": "campaign-official-doubao-seed-2.0-pro",
|
| 34 |
+
"gpt-5.4": "campaign-official-gpt-5.4-aime65536-highlevel-llmjudge-safety-strict-llmjudge",
|
| 35 |
+
"safe-innospark": "campaign-official-safe-innospark-aime65536-highlevel-llmjudge",
|
| 36 |
+
"innospark-235b": (
|
| 37 |
+
"campaign-official-innospark-235b-aime65536-highlevel-llmjudge-"
|
| 38 |
+
"safety-fill-targeted-safety-strict-llmjudge"
|
| 39 |
+
),
|
| 40 |
+
"glm-5.1": "campaign-official-glm-5.1",
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
REQUIRED_COLUMNS = ["ID", "Question_Chinese", "Question_English", "Answer"]
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def parse_args() -> argparse.Namespace:
|
| 47 |
+
parser = argparse.ArgumentParser(
|
| 48 |
+
description=(
|
| 49 |
+
"Replace the ELBench SATAs sampled file with corrected references and "
|
| 50 |
+
"rejudge existing SATAs model outputs."
|
| 51 |
+
)
|
| 52 |
+
)
|
| 53 |
+
parser.add_argument("--config-root", default="configs")
|
| 54 |
+
parser.add_argument("--corrected-path", type=Path, required=True)
|
| 55 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 56 |
+
return parser.parse_args()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def main() -> None:
|
| 60 |
+
args = parse_args()
|
| 61 |
+
config = load_project_config(args.config_root)
|
| 62 |
+
current_path = _resolve_current_satas_path(config)
|
| 63 |
+
corrected_df = _read_first_sheet(args.corrected_path)
|
| 64 |
+
current_df = _read_first_sheet(current_path)
|
| 65 |
+
sampled_df = _build_corrected_sample(current_df=current_df, corrected_df=corrected_df)
|
| 66 |
+
|
| 67 |
+
if args.dry_run:
|
| 68 |
+
print(
|
| 69 |
+
json.dumps(
|
| 70 |
+
{
|
| 71 |
+
"current_path": str(current_path),
|
| 72 |
+
"corrected_path": str(args.corrected_path),
|
| 73 |
+
"current_rows": len(current_df),
|
| 74 |
+
"corrected_rows": len(corrected_df),
|
| 75 |
+
"sampled_rows": len(sampled_df),
|
| 76 |
+
"changed_answers": int(
|
| 77 |
+
(
|
| 78 |
+
current_df["Answer"].fillna("").astype(str).to_numpy()
|
| 79 |
+
!= sampled_df["Answer"].fillna("").astype(str).to_numpy()
|
| 80 |
+
).sum()
|
| 81 |
+
),
|
| 82 |
+
},
|
| 83 |
+
ensure_ascii=False,
|
| 84 |
+
indent=2,
|
| 85 |
+
)
|
| 86 |
+
)
|
| 87 |
+
return
|
| 88 |
+
|
| 89 |
+
_replace_satas_file(current_path=current_path, sampled_df=sampled_df)
|
| 90 |
+
sample_by_id = _load_corrected_satas_samples(config)
|
| 91 |
+
results = {}
|
| 92 |
+
for model_id, run_id in MODELS_AND_RUNS.items():
|
| 93 |
+
results[model_id] = asyncio.run(
|
| 94 |
+
_rejudge_model_satas(
|
| 95 |
+
output_root=config.app.output_root,
|
| 96 |
+
model_id=model_id,
|
| 97 |
+
run_id=run_id,
|
| 98 |
+
sample_by_id=sample_by_id,
|
| 99 |
+
)
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
manifest = {
|
| 103 |
+
"generated_at": datetime.now().isoformat(),
|
| 104 |
+
"current_path": str(current_path),
|
| 105 |
+
"corrected_path": str(args.corrected_path),
|
| 106 |
+
"sampled_rows": len(sampled_df),
|
| 107 |
+
"changed_answers": int(
|
| 108 |
+
(
|
| 109 |
+
current_df["Answer"].fillna("").astype(str).to_numpy()
|
| 110 |
+
!= sampled_df["Answer"].fillna("").astype(str).to_numpy()
|
| 111 |
+
).sum()
|
| 112 |
+
),
|
| 113 |
+
"models": results,
|
| 114 |
+
}
|
| 115 |
+
audit_dir = ROOT / "results" / "audit-judge-integrity"
|
| 116 |
+
audit_dir.mkdir(parents=True, exist_ok=True)
|
| 117 |
+
manifest_path = audit_dir / "satas_reference_rejudge_manifest.json"
|
| 118 |
+
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 119 |
+
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _resolve_current_satas_path(config) -> Path:
|
| 123 |
+
registry = FileRegistry(config)
|
| 124 |
+
items = registry.resolve(source_files={"SATAs.xlsx"})
|
| 125 |
+
if len(items) != 1:
|
| 126 |
+
raise RuntimeError(f"Expected one SATAs registry item, got {len(items)}")
|
| 127 |
+
return items[0].path
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _read_first_sheet(path: Path) -> pd.DataFrame:
|
| 131 |
+
if not path.exists():
|
| 132 |
+
raise FileNotFoundError(path)
|
| 133 |
+
df = pd.read_excel(path, sheet_name=0)
|
| 134 |
+
missing = [column for column in REQUIRED_COLUMNS if column not in df.columns]
|
| 135 |
+
if missing:
|
| 136 |
+
raise ValueError(f"{path} is missing columns: {missing}")
|
| 137 |
+
df = df.loc[:, REQUIRED_COLUMNS].copy()
|
| 138 |
+
if df[REQUIRED_COLUMNS].isna().any(axis=1).any():
|
| 139 |
+
bad_count = int(df[REQUIRED_COLUMNS].isna().any(axis=1).sum())
|
| 140 |
+
raise ValueError(f"{path} has {bad_count} rows with empty required fields")
|
| 141 |
+
if df["ID"].duplicated().any():
|
| 142 |
+
duplicate_ids = df.loc[df["ID"].duplicated(keep=False), "ID"].astype(str).tolist()
|
| 143 |
+
raise ValueError(f"{path} has duplicate IDs: {duplicate_ids[:10]}")
|
| 144 |
+
return df
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _build_corrected_sample(*, current_df: pd.DataFrame, corrected_df: pd.DataFrame) -> pd.DataFrame:
|
| 148 |
+
current_ids = current_df["ID"].astype(str).tolist()
|
| 149 |
+
corrected_by_id = corrected_df.assign(ID=corrected_df["ID"].astype(str)).set_index("ID", drop=False)
|
| 150 |
+
missing = [sample_id for sample_id in current_ids if sample_id not in corrected_by_id.index]
|
| 151 |
+
if missing:
|
| 152 |
+
raise ValueError(f"Corrected SATAs file is missing sampled IDs: {missing[:10]}")
|
| 153 |
+
sampled_df = corrected_by_id.loc[current_ids, REQUIRED_COLUMNS].reset_index(drop=True)
|
| 154 |
+
for column in ["Question_Chinese", "Question_English"]:
|
| 155 |
+
left = current_df[column].fillna("").astype(str).reset_index(drop=True)
|
| 156 |
+
right = sampled_df[column].fillna("").astype(str).reset_index(drop=True)
|
| 157 |
+
if not left.equals(right):
|
| 158 |
+
changed_ids = [
|
| 159 |
+
current_ids[index]
|
| 160 |
+
for index, (left_value, right_value) in enumerate(zip(left, right))
|
| 161 |
+
if left_value != right_value
|
| 162 |
+
]
|
| 163 |
+
raise ValueError(f"Corrected SATAs changed {column} for sampled IDs: {changed_ids[:10]}")
|
| 164 |
+
return sampled_df
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _replace_satas_file(*, current_path: Path, sampled_df: pd.DataFrame) -> None:
|
| 168 |
+
backup_dir = ROOT / "results" / "audit-judge-integrity" / "satas_backups"
|
| 169 |
+
backup_dir.mkdir(parents=True, exist_ok=True)
|
| 170 |
+
backup_path = backup_dir / f"{current_path.stem}.before_reference_fix.xlsx"
|
| 171 |
+
if not backup_path.exists():
|
| 172 |
+
shutil.copy2(current_path, backup_path)
|
| 173 |
+
sampled_df.to_excel(current_path, index=False)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _load_corrected_satas_samples(config) -> dict[str, Sample]:
|
| 177 |
+
registry = FileRegistry(config)
|
| 178 |
+
items = registry.resolve(source_files={"SATAs.xlsx"})
|
| 179 |
+
if len(items) != 1:
|
| 180 |
+
raise RuntimeError(f"Expected one SATAs registry item, got {len(items)}")
|
| 181 |
+
loader = LoaderFactory.create(items[0].entry.loader_name)
|
| 182 |
+
samples = list(loader.iter_samples(items[0]))
|
| 183 |
+
if len(samples) != 150:
|
| 184 |
+
raise RuntimeError(f"Expected 150 sampled SATAs records, got {len(samples)}")
|
| 185 |
+
return {sample.sample_id: sample for sample in samples}
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
async def _rejudge_model_satas(
|
| 189 |
+
*,
|
| 190 |
+
output_root: Path,
|
| 191 |
+
model_id: str,
|
| 192 |
+
run_id: str,
|
| 193 |
+
sample_by_id: dict[str, Sample],
|
| 194 |
+
) -> dict[str, Any]:
|
| 195 |
+
paths = OutputPaths.build(output_root, run_id, model_id)
|
| 196 |
+
if not paths.judged_path.exists():
|
| 197 |
+
raise FileNotFoundError(paths.judged_path)
|
| 198 |
+
if not paths.raw_path.exists():
|
| 199 |
+
raise FileNotFoundError(paths.raw_path)
|
| 200 |
+
|
| 201 |
+
judged_before = _load_jsonl(paths.judged_path)
|
| 202 |
+
raw_before = _load_jsonl(paths.raw_path)
|
| 203 |
+
judged_after, judged_stats = await _rewrite_records_with_new_satas_judgment(
|
| 204 |
+
records=judged_before,
|
| 205 |
+
sample_by_id=sample_by_id,
|
| 206 |
+
update_judgment=True,
|
| 207 |
+
)
|
| 208 |
+
raw_after, raw_stats = await _rewrite_records_with_new_satas_judgment(
|
| 209 |
+
records=raw_before,
|
| 210 |
+
sample_by_id=sample_by_id,
|
| 211 |
+
update_judgment=False,
|
| 212 |
+
)
|
| 213 |
+
if judged_stats["matched"] != 150:
|
| 214 |
+
raise RuntimeError(f"{model_id} judged SATAs count is {judged_stats['matched']}, expected 150")
|
| 215 |
+
if raw_stats["matched"] != 150:
|
| 216 |
+
raise RuntimeError(f"{model_id} raw SATAs count is {raw_stats['matched']}, expected 150")
|
| 217 |
+
|
| 218 |
+
_write_jsonl(paths.judged_path, judged_after)
|
| 219 |
+
_write_jsonl(paths.raw_path, raw_after)
|
| 220 |
+
summary = build_summary(paths.judged_path, paths.failures_path)
|
| 221 |
+
paths.summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 222 |
+
return {
|
| 223 |
+
"run_id": run_id,
|
| 224 |
+
"judged_total": len(judged_after),
|
| 225 |
+
"raw_total": len(raw_after),
|
| 226 |
+
"satas_judged": judged_stats,
|
| 227 |
+
"satas_raw": raw_stats,
|
| 228 |
+
"summary_total_failures": summary.get("total_failures"),
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
async def _rewrite_records_with_new_satas_judgment(
|
| 233 |
+
*,
|
| 234 |
+
records: list[dict[str, Any]],
|
| 235 |
+
sample_by_id: dict[str, Sample],
|
| 236 |
+
update_judgment: bool,
|
| 237 |
+
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
| 238 |
+
judge = TeachingHarmJudge()
|
| 239 |
+
changed_judgments = 0
|
| 240 |
+
matched = 0
|
| 241 |
+
pass_count = 0
|
| 242 |
+
after: list[dict[str, Any]] = []
|
| 243 |
+
for record in records:
|
| 244 |
+
if record.get("task") != "teaching_harm" and record.get("source_file") != "SATAs.xlsx":
|
| 245 |
+
after.append(record)
|
| 246 |
+
continue
|
| 247 |
+
sample = sample_by_id.get(str(record.get("sample_id") or ""))
|
| 248 |
+
if sample is None:
|
| 249 |
+
raise RuntimeError(f"Cannot map SATAs record sample_id={record.get('sample_id')!r}")
|
| 250 |
+
matched += 1
|
| 251 |
+
updated = dict(record)
|
| 252 |
+
updated["source_path"] = sample.source_path
|
| 253 |
+
updated["prompt"] = sample.prompt
|
| 254 |
+
updated["reference"] = sample.reference
|
| 255 |
+
metadata = dict(updated.get("metadata") or {})
|
| 256 |
+
metadata.update(sample.metadata)
|
| 257 |
+
updated["metadata"] = metadata
|
| 258 |
+
if update_judgment:
|
| 259 |
+
previous = (updated.get("judge_result"), updated.get("score"), updated.get("judge_metadata"))
|
| 260 |
+
response = ModelResponse(
|
| 261 |
+
text=updated.get("model_response"),
|
| 262 |
+
raw_payload=updated.get("raw_response") if isinstance(updated.get("raw_response"), dict) else {},
|
| 263 |
+
latency_ms=updated.get("latency_ms"),
|
| 264 |
+
retry_count=int(updated.get("retry_count") or 0),
|
| 265 |
+
)
|
| 266 |
+
judgment = await judge.judge(sample, response)
|
| 267 |
+
updated["judge_result"] = judgment.judge_result
|
| 268 |
+
updated["score"] = judgment.score
|
| 269 |
+
updated["judge_reason"] = judgment.judge_reason
|
| 270 |
+
updated["judge_metadata"] = judgment.judge_metadata
|
| 271 |
+
current = (updated.get("judge_result"), updated.get("score"), updated.get("judge_metadata"))
|
| 272 |
+
if current != previous:
|
| 273 |
+
changed_judgments += 1
|
| 274 |
+
if updated.get("judge_result") == "pass":
|
| 275 |
+
pass_count += 1
|
| 276 |
+
after.append(updated)
|
| 277 |
+
return after, {"matched": matched, "changed_judgments": changed_judgments, "pass_count": pass_count}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def _load_jsonl(path: Path) -> list[dict[str, Any]]:
|
| 281 |
+
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
| 285 |
+
temp_path = path.with_suffix(path.suffix + ".tmp")
|
| 286 |
+
with temp_path.open("w", encoding="utf-8") as handle:
|
| 287 |
+
for record in records:
|
| 288 |
+
handle.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
|
| 289 |
+
temp_path.replace(path)
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
if __name__ == "__main__":
|
| 293 |
+
main()
|
scripts/run_benchmark.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
import sys
|
| 3 |
+
|
| 4 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 5 |
+
SRC = ROOT / "src"
|
| 6 |
+
if str(SRC) not in sys.path:
|
| 7 |
+
sys.path.insert(0, str(SRC))
|
| 8 |
+
|
| 9 |
+
from elbench.cli import main
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
if __name__ == "__main__":
|
| 13 |
+
main()
|
| 14 |
+
|
src/elbench/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Education benchmark framework."""
|
src/elbench/basic_education_runtime/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Internal runtime for ELBench basic education scenarios."""
|
| 2 |
+
|
src/elbench/basic_education_runtime/agent.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langchain.chat_models.base import BaseChatModel
|
| 2 |
+
from langgraph.prebuilt.chat_agent_executor import AgentState
|
| 3 |
+
from langgraph.graph import StateGraph, START, END
|
| 4 |
+
from langgraph.graph.state import CompiledStateGraph
|
| 5 |
+
from langchain_core.messages import AIMessage, HumanMessage
|
| 6 |
+
|
| 7 |
+
from typing import Any, Dict, List, Callable, Optional, Tuple, Awaitable
|
| 8 |
+
from tenacity import retry, stop_after_attempt, wait_fixed
|
| 9 |
+
|
| 10 |
+
from elbench.basic_education_runtime.entity import AgentConfig
|
| 11 |
+
from elbench.basic_education_runtime.utils import content_to_text, replace_prompt, remove_think
|
| 12 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 13 |
+
|
| 14 |
+
import asyncio
|
| 15 |
+
import copy
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _init_agent_from_dict(
|
| 19 |
+
ac: AgentConfig,
|
| 20 |
+
model_map: Dict[str, BaseChatModel],
|
| 21 |
+
agent_name: str,
|
| 22 |
+
dynamic_prompt_map: Optional[Dict[str, str]] = None,
|
| 23 |
+
) -> Callable[..., Awaitable[Dict[str, List[Any]]]]:
|
| 24 |
+
if dynamic_prompt_map is not None:
|
| 25 |
+
ac_prompt = replace_prompt(ac.prompt, dynamic_prompt_map)
|
| 26 |
+
else:
|
| 27 |
+
ac_prompt = copy.deepcopy(ac.prompt) # fix shallow copy瀵艰嚧鐨勬暟鎹敊璇?
|
| 28 |
+
m = model_map[ac.model]
|
| 29 |
+
|
| 30 |
+
@retry(
|
| 31 |
+
stop=stop_after_attempt(CONFIG.globals.retry.attempt),
|
| 32 |
+
wait=wait_fixed(CONFIG.globals.retry.interval),
|
| 33 |
+
)
|
| 34 |
+
async def chatbot(state: AgentState) -> Dict[str, List[Any]]:
|
| 35 |
+
if state["messages"] == []:
|
| 36 |
+
n_m = ac_prompt
|
| 37 |
+
else:
|
| 38 |
+
n_m = []
|
| 39 |
+
for item in state["messages"]:
|
| 40 |
+
content = content_to_text(remove_think(item.content))
|
| 41 |
+
if not content.strip():
|
| 42 |
+
continue
|
| 43 |
+
if item.name == agent_name:
|
| 44 |
+
item = AIMessage(content=content, type="ai") # type: ignore
|
| 45 |
+
else:
|
| 46 |
+
item = HumanMessage(content=content, type="human") # type: ignore
|
| 47 |
+
n_m.append(item)
|
| 48 |
+
if len(n_m) > ac.memory.keep_turns * 2 + 1:
|
| 49 |
+
n_m = n_m[-ac.memory.keep_turns * 2 - 1 :]
|
| 50 |
+
n_m = ac_prompt + n_m
|
| 51 |
+
r = await asyncio.wait_for( # type: ignore
|
| 52 |
+
m.ainvoke(n_m),
|
| 53 |
+
timeout=CONFIG.globals.model_call_timeout_seconds,
|
| 54 |
+
)
|
| 55 |
+
r.name = agent_name
|
| 56 |
+
return {"messages": [r]}
|
| 57 |
+
|
| 58 |
+
return chatbot
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def init_agent_map_from_dict(
|
| 62 |
+
model_map: Dict[str, BaseChatModel],
|
| 63 |
+
dynamic_prompt_map: Optional[Dict[str, str]] = None,
|
| 64 |
+
) -> Tuple[Dict[str, Tuple[CompiledStateGraph, AgentConfig]], Optional[Dict[str, str]]]:
|
| 65 |
+
result = {}
|
| 66 |
+
acs = CONFIG.agents
|
| 67 |
+
for k, v in acs.items():
|
| 68 |
+
ac = v
|
| 69 |
+
if ac.memory.enable:
|
| 70 |
+
# conn = sqlite3.connect(f"{k}_checkpoint.sqlite", check_same_thread=False)
|
| 71 |
+
# memory = SqliteSaver(conn)
|
| 72 |
+
memory = True
|
| 73 |
+
else:
|
| 74 |
+
memory = None
|
| 75 |
+
model = _init_agent_from_dict(ac, model_map, k, dynamic_prompt_map)
|
| 76 |
+
graph = StateGraph(AgentState)
|
| 77 |
+
graph.add_node("agent", model)
|
| 78 |
+
graph.add_edge(START, "agent")
|
| 79 |
+
graph.add_edge("agent", END)
|
| 80 |
+
result[k] = (graph.compile(checkpointer=memory), ac)
|
| 81 |
+
return result, dynamic_prompt_map
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
init_agent_map = init_agent_map_from_dict
|
| 85 |
+
|
src/elbench/basic_education_runtime/assets/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Assets used by the internal basic education runtime."""
|
src/elbench/basic_education_runtime/assets/fonts/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Font assets for the internal basic education runtime."""
|
src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:22d82510f7ac3b342668055e1c6e935b80962e86abd0f2e3c1ad9919a2419b21
|
| 3 |
+
size 22515080
|
src/elbench/basic_education_runtime/cli/draw/__init__.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@click.command(help="Draw Agent workflow.")
|
| 6 |
+
@click.option(
|
| 7 |
+
"--config", type=click.Path(exists=True), help="Path to the configuration file"
|
| 8 |
+
)
|
| 9 |
+
@click.option("--debug", is_flag=True, help="Enable debug mode")
|
| 10 |
+
def draw(config, debug=False):
|
| 11 |
+
set_debug(debug)
|
| 12 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
config = Path(config)
|
| 16 |
+
load_conf(config)
|
| 17 |
+
|
| 18 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 19 |
+
from elbench.basic_education_runtime.model import init_model_map
|
| 20 |
+
from elbench.basic_education_runtime.agent import init_agent_map
|
| 21 |
+
from elbench.basic_education_runtime.directions import apply_agent_direction
|
| 22 |
+
|
| 23 |
+
# from langchain_core.runnables.graph import CurveStyle, NodeStyles, MermaidDrawMethod
|
| 24 |
+
|
| 25 |
+
import asyncio
|
| 26 |
+
|
| 27 |
+
models = init_model_map()
|
| 28 |
+
task = CONFIG.tasks.variables[0]
|
| 29 |
+
agents, _ = init_agent_map(models, task)
|
| 30 |
+
agent, _ = asyncio.run(apply_agent_direction(agents, task=task))
|
| 31 |
+
png = agent.get_graph().draw_mermaid_png()
|
| 32 |
+
with open(f"{config.stem}.png", "wb") as wb:
|
| 33 |
+
wb.write(png)
|
| 34 |
+
# print(png)
|
| 35 |
+
|
src/elbench/basic_education_runtime/cli/eval/__init__.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Dict, Any
|
| 5 |
+
|
| 6 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 7 |
+
from tenacity import RetryError
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@click.command(help="Evaluate the performance of a model on a dataset")
|
| 11 |
+
@click.option(
|
| 12 |
+
"--config",
|
| 13 |
+
type=click.Path(exists=True),
|
| 14 |
+
required=True,
|
| 15 |
+
help="Path to the configuration file",
|
| 16 |
+
)
|
| 17 |
+
@click.option("--debug", default=False, help="Debug Mode", is_flag=True)
|
| 18 |
+
@click.option("--avg/--no-avg", default=True, help="Calculate the average score")
|
| 19 |
+
def eval(config: Path, debug: bool, avg: bool):
|
| 20 |
+
set_debug(debug)
|
| 21 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 22 |
+
|
| 23 |
+
load_conf(config)
|
| 24 |
+
eval_logic(avg)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def eval_logic(avg: bool):
|
| 28 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 29 |
+
|
| 30 |
+
input_dir = CONFIG.globals.memory.path
|
| 31 |
+
import asyncio
|
| 32 |
+
from elbench.basic_education_runtime.evaluation import evaluate
|
| 33 |
+
from elbench.basic_education_runtime.model import init_chat_model_from_dict
|
| 34 |
+
|
| 35 |
+
from elbench.basic_education_runtime.entity import ExportFormat
|
| 36 |
+
from tqdm.asyncio import tqdm
|
| 37 |
+
import json
|
| 38 |
+
|
| 39 |
+
input_dir = Path(input_dir)
|
| 40 |
+
|
| 41 |
+
eval_path = input_dir / "eval"
|
| 42 |
+
eval_path.mkdir(exist_ok=True)
|
| 43 |
+
|
| 44 |
+
sem = asyncio.Semaphore(CONFIG.globals.concurrency)
|
| 45 |
+
|
| 46 |
+
async def eval_task(model, file: Path) -> Dict[str, Any]:
|
| 47 |
+
async with sem:
|
| 48 |
+
ef = ExportFormat.from_json_file(file)
|
| 49 |
+
try:
|
| 50 |
+
eval = await evaluate(model, ef)
|
| 51 |
+
with open(eval_path / file.name, "w", encoding="utf8") as f:
|
| 52 |
+
json.dump(eval, f, ensure_ascii=False, indent=4)
|
| 53 |
+
return eval
|
| 54 |
+
except RetryError as e:
|
| 55 |
+
print(f"Error evaluating {file}", e.last_attempt.exception())
|
| 56 |
+
return {}
|
| 57 |
+
|
| 58 |
+
async def main():
|
| 59 |
+
assert CONFIG.evaluation
|
| 60 |
+
model = init_chat_model_from_dict(CONFIG.models[CONFIG.evaluation.model])
|
| 61 |
+
|
| 62 |
+
to_eval_files = list(input_dir.glob("*.json"))
|
| 63 |
+
task_ids = [file.stem for file in to_eval_files]
|
| 64 |
+
eval_tasks = []
|
| 65 |
+
for file in to_eval_files:
|
| 66 |
+
eval_tasks.append(eval_task(model, file))
|
| 67 |
+
|
| 68 |
+
evals = await tqdm.gather(*eval_tasks)
|
| 69 |
+
|
| 70 |
+
csv_utf8 = open(
|
| 71 |
+
eval_path / f"{CONFIG.evaluation.name}.csv", "w", encoding="utf-8"
|
| 72 |
+
)
|
| 73 |
+
# csv_gbk = open(eval_path / f"{CONFIG.evaluation.name}-gbk.csv", "w", encoding="gbk")
|
| 74 |
+
|
| 75 |
+
metric_fields = _metric_fields_from_evals(evals)
|
| 76 |
+
if not metric_fields:
|
| 77 |
+
csv_utf8.write("task_id\n")
|
| 78 |
+
csv_utf8.close()
|
| 79 |
+
raise RuntimeError("No successful evaluation outputs were produced.")
|
| 80 |
+
|
| 81 |
+
title = ["task_id"]
|
| 82 |
+
for field in metric_fields:
|
| 83 |
+
title.append(field)
|
| 84 |
+
|
| 85 |
+
if avg:
|
| 86 |
+
title.append("avg")
|
| 87 |
+
|
| 88 |
+
csv_utf8.write(",".join(title) + "\n")
|
| 89 |
+
# csv_gbk.write(",".join(title) + "\n")
|
| 90 |
+
|
| 91 |
+
if avg:
|
| 92 |
+
row = len(task_ids) + 1
|
| 93 |
+
metric_count = len(metric_fields)
|
| 94 |
+
col = metric_count + 1
|
| 95 |
+
|
| 96 |
+
matrix = [[0.0] * col for _ in range(row)]
|
| 97 |
+
|
| 98 |
+
# 缁熻鏁版嵁骞惰绠楁瘡琛屽钩鍧囧€?
|
| 99 |
+
for idx, (task_id, eval) in enumerate(zip(task_ids, evals)):
|
| 100 |
+
contents = [task_id]
|
| 101 |
+
for sub_idx, field in enumerate(metric_fields):
|
| 102 |
+
raw_value = eval.get(field, 0)
|
| 103 |
+
v = float(raw_value)
|
| 104 |
+
matrix[idx][sub_idx] = v
|
| 105 |
+
contents.append(f"{raw_value}")
|
| 106 |
+
sum = 0
|
| 107 |
+
for i in matrix[idx][:-1]:
|
| 108 |
+
sum += i
|
| 109 |
+
# 鏈€鍚庝竴鍒楃殑鏁板瓧 = 姣忓垪鐨勫拰闄や互鍒楁暟-1
|
| 110 |
+
matrix[idx][col - 1] = sum / metric_count
|
| 111 |
+
contents.append(f"{matrix[idx][col - 1]:.2f}")
|
| 112 |
+
csv_utf8.write(",".join(contents) + "\n")
|
| 113 |
+
# csv_gbk.write(",".join(contents) + "\n")
|
| 114 |
+
# 璁$畻姣忓垪鐨勫钩鍧囧€?
|
| 115 |
+
for col_idx in range(col):
|
| 116 |
+
# print(matrix)
|
| 117 |
+
sum = 0
|
| 118 |
+
# 璁$畻姣忎竴鍒楅櫎鍘绘渶鍚庝竴涓厓绱犵殑鍜?
|
| 119 |
+
for row_idx in range(row - 1):
|
| 120 |
+
sum += matrix[row_idx][col_idx]
|
| 121 |
+
matrix[-1][col_idx] = sum / (row - 1)
|
| 122 |
+
|
| 123 |
+
write_str = ["%.2f" % i for i in matrix[-1]]
|
| 124 |
+
write_str.insert(0, "Avg")
|
| 125 |
+
# 鍐欏叆鏈€鍚庝竴琛岀殑骞冲潎鍊?
|
| 126 |
+
csv_utf8.write(",".join(write_str) + "\n")
|
| 127 |
+
# csv_gbk.write(",".join(write_str) + "\n")
|
| 128 |
+
else:
|
| 129 |
+
for task_id, eval in zip(task_ids, evals):
|
| 130 |
+
contents = [task_id]
|
| 131 |
+
for field in metric_fields:
|
| 132 |
+
contents.append(f"{eval.get(field, 0)}")
|
| 133 |
+
csv_utf8.write(",".join(contents) + "\n")
|
| 134 |
+
# csv_gbk.write(",".join(contents) + "\n")
|
| 135 |
+
|
| 136 |
+
csv_utf8.close()
|
| 137 |
+
# csv_gbk.close()
|
| 138 |
+
|
| 139 |
+
asyncio.run(main())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _metric_fields_from_evals(evals: list[Dict[str, Any]]) -> list[str]:
|
| 143 |
+
for item in evals:
|
| 144 |
+
if item:
|
| 145 |
+
return list(item.keys())
|
| 146 |
+
return []
|
| 147 |
+
|
src/elbench/basic_education_runtime/cli/export/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
|
| 3 |
+
from elbench.basic_education_runtime.cli.export.json_ import json
|
| 4 |
+
|
| 5 |
+
try:
|
| 6 |
+
from elbench.basic_education_runtime.cli.export.label_studio import label_studio
|
| 7 |
+
except Exception: # optional command; do not block core pipeline/export-json flow
|
| 8 |
+
label_studio = None
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@click.group(help="Export chat databases")
|
| 12 |
+
def export():
|
| 13 |
+
pass
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
export.add_command(json)
|
| 17 |
+
if label_studio is not None:
|
| 18 |
+
export.add_command(label_studio)
|
| 19 |
+
|
src/elbench/basic_education_runtime/cli/export/const/label_studio.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Sequence
|
| 2 |
+
from elbench.basic_education_runtime.entity import FormatField
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
TEMPLATE_template = """<View>
|
| 6 |
+
<Style>
|
| 7 |
+
.container {
|
| 8 |
+
display: flex;
|
| 9 |
+
justify-content: space-between;
|
| 10 |
+
margin: 0 auto;
|
| 11 |
+
padding: 20px;
|
| 12 |
+
background-color: #ffffff;
|
| 13 |
+
border-radius: 5px;
|
| 14 |
+
box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.1), 0 6px 20px 0 rgba(0, 0, 0, 0.1);
|
| 15 |
+
max-width: 800px;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
.border {
|
| 19 |
+
border-style: solid;
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
.text-block {
|
| 23 |
+
flex: 1;
|
| 24 |
+
margin-right: 20px;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
.assessment-items-container {
|
| 28 |
+
flex: 1;
|
| 29 |
+
display: flex;
|
| 30 |
+
flex-direction: column;
|
| 31 |
+
gap: 20px;
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
.assessment-item {
|
| 35 |
+
background-color: rgba(44, 62, 80, 0.6);
|
| 36 |
+
padding: 1px;
|
| 37 |
+
border-radius: 5px;
|
| 38 |
+
box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.1), 0 3px 10px 0 rgba(0, 0, 0, 0.1);
|
| 39 |
+
color: #ffffff;
|
| 40 |
+
word-wrap: break-word;
|
| 41 |
+
}
|
| 42 |
+
</Style>
|
| 43 |
+
<View className="container">
|
| 44 |
+
<View className="text-block">
|
| 45 |
+
<Paragraphs name="dialogue" value="$messages" layout="dialogue" nameKey="role" textKey="content" />
|
| 46 |
+
</View>
|
| 47 |
+
<View className="assessment-item-container">
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
TEMPLATE_SUFFIX = """
|
| 51 |
+
</View>
|
| 52 |
+
</View>
|
| 53 |
+
</View>
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def generate_labeling(fields: Sequence[FormatField]) -> str:
|
| 58 |
+
compoents: list[str] = []
|
| 59 |
+
for field in fields:
|
| 60 |
+
name = field.field
|
| 61 |
+
template = f' <Header value="{name}" size="8"/><View className="assessment-item">%s</View>\n'
|
| 62 |
+
if field.type == "int":
|
| 63 |
+
if field.max is not None:
|
| 64 |
+
compoents.append(template % f'<Rating name="{name}" maxRating="{field.max}" toName="dialogue" />')
|
| 65 |
+
else:
|
| 66 |
+
compoents.append(template % f'<Rating name="{name}" maxRating="5" toName="dialogue" />')
|
| 67 |
+
elif field.type == "float":
|
| 68 |
+
if field.max is not None:
|
| 69 |
+
compoents.append(template % f'<Number name="{name}" toName="dialogue" max="{field.max}" />')
|
| 70 |
+
else:
|
| 71 |
+
compoents.append(template % f'<Number name="{name}" toName="dialogue" />')
|
| 72 |
+
elif field.type == "str":
|
| 73 |
+
compoents.append(template % f'<TextArea name="{name}" toName="dialogue" />')
|
| 74 |
+
elif field.type == "bool":
|
| 75 |
+
compoents.append(template % f'<Choices name="{name}" toName="dialogue" showInline="true" choice="single-radio"><Choice value="Yes"/><Choice value="No"/></Choices>')
|
| 76 |
+
elif field.type == "dict":
|
| 77 |
+
children = generate_labeling(field.items)
|
| 78 |
+
compoents.append(template % f'<View className="border">{"".join(children)}</View>')
|
| 79 |
+
else:
|
| 80 |
+
raise NotImplementedError
|
| 81 |
+
return "".join(compoents)
|
| 82 |
+
|
| 83 |
+
def generate_label_studio_interface(fields: Sequence[FormatField]) -> str:
|
| 84 |
+
return TEMPLATE_template + generate_labeling(fields) + TEMPLATE_SUFFIX
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
|
src/elbench/basic_education_runtime/cli/export/exporter/json_.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sqlite3
|
| 2 |
+
from typing import Tuple, Dict, Any
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import json
|
| 5 |
+
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
| 6 |
+
from langgraph.checkpoint.base import Checkpoint
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _to_jsonable(value: Any) -> Any:
|
| 10 |
+
return json.loads(json.dumps(value, ensure_ascii=False, default=str))
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _stringify_block_text(value: Any) -> str:
|
| 14 |
+
if value is None:
|
| 15 |
+
return ""
|
| 16 |
+
if isinstance(value, str):
|
| 17 |
+
return value
|
| 18 |
+
if isinstance(value, list):
|
| 19 |
+
parts: list[str] = []
|
| 20 |
+
for item in value:
|
| 21 |
+
if isinstance(item, dict):
|
| 22 |
+
text = item.get("text")
|
| 23 |
+
if text not in (None, ""):
|
| 24 |
+
parts.append(str(text))
|
| 25 |
+
continue
|
| 26 |
+
if item not in (None, ""):
|
| 27 |
+
parts.append(str(item))
|
| 28 |
+
return "\n".join(parts).strip()
|
| 29 |
+
if isinstance(value, dict):
|
| 30 |
+
text = value.get("text")
|
| 31 |
+
if text not in (None, ""):
|
| 32 |
+
return str(text)
|
| 33 |
+
return json.dumps(_to_jsonable(value), ensure_ascii=False)
|
| 34 |
+
return str(value)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _split_reasoning_and_response(content: Any) -> tuple[str, str]:
|
| 38 |
+
if isinstance(content, str):
|
| 39 |
+
if "</think>" in content:
|
| 40 |
+
content_split = content.split("</think>", 1)
|
| 41 |
+
return content_split[0].strip(), content_split[1].strip()
|
| 42 |
+
return "", content.strip()
|
| 43 |
+
|
| 44 |
+
if isinstance(content, list):
|
| 45 |
+
reasoning_parts: list[str] = []
|
| 46 |
+
response_parts: list[str] = []
|
| 47 |
+
for block in content:
|
| 48 |
+
if isinstance(block, dict):
|
| 49 |
+
block_type = str(block.get("type", "")).strip().lower()
|
| 50 |
+
if block_type == "reasoning":
|
| 51 |
+
reasoning_text = _stringify_block_text(block.get("summary"))
|
| 52 |
+
if not reasoning_text:
|
| 53 |
+
reasoning_text = _stringify_block_text(block.get("text"))
|
| 54 |
+
if reasoning_text:
|
| 55 |
+
reasoning_parts.append(reasoning_text)
|
| 56 |
+
continue
|
| 57 |
+
if block_type in {"text", "output_text"}:
|
| 58 |
+
text = _stringify_block_text(block.get("text"))
|
| 59 |
+
if text:
|
| 60 |
+
response_parts.append(text)
|
| 61 |
+
continue
|
| 62 |
+
fallback = _stringify_block_text(block)
|
| 63 |
+
if fallback:
|
| 64 |
+
response_parts.append(fallback)
|
| 65 |
+
continue
|
| 66 |
+
|
| 67 |
+
fallback = _stringify_block_text(block)
|
| 68 |
+
if fallback:
|
| 69 |
+
response_parts.append(fallback)
|
| 70 |
+
return "\n".join(reasoning_parts).strip(), "\n".join(response_parts).strip()
|
| 71 |
+
|
| 72 |
+
response = _stringify_block_text(content).strip()
|
| 73 |
+
return "", response
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
async def aexport_json(input_path: Path) -> Tuple[Path, Dict[str, Any]]:
|
| 77 |
+
conn = sqlite3.connect(input_path)
|
| 78 |
+
cursor = conn.cursor()
|
| 79 |
+
sql = "select checkpoint_ns, checkpoint_id, parent_checkpoint_id, checkpoint from checkpoints"
|
| 80 |
+
cursor.execute(sql)
|
| 81 |
+
results = cursor.fetchall()[-1]
|
| 82 |
+
cns, cid, pcid, c = results
|
| 83 |
+
jps = JsonPlusSerializer()
|
| 84 |
+
checkpoint: Checkpoint = jps.loads_typed(("msgpack", c))
|
| 85 |
+
messages = []
|
| 86 |
+
for m in checkpoint.get("channel_values")["messages"]:
|
| 87 |
+
if m.name is None:
|
| 88 |
+
continue
|
| 89 |
+
reasoning, response = _split_reasoning_and_response(getattr(m, "content", None))
|
| 90 |
+
item = {"role": m.name, "content": response, "reasoning": reasoning}
|
| 91 |
+
usage_metadata = getattr(m, "usage_metadata", None)
|
| 92 |
+
response_metadata = getattr(m, "response_metadata", None)
|
| 93 |
+
additional_kwargs = getattr(m, "additional_kwargs", None)
|
| 94 |
+
if usage_metadata not in (None, {}):
|
| 95 |
+
item["usage_metadata"] = _to_jsonable(usage_metadata)
|
| 96 |
+
if response_metadata not in (None, {}):
|
| 97 |
+
item["response_metadata"] = _to_jsonable(response_metadata)
|
| 98 |
+
if additional_kwargs not in (None, {}):
|
| 99 |
+
item["additional_kwargs"] = _to_jsonable(additional_kwargs)
|
| 100 |
+
messages.append(item)
|
| 101 |
+
|
| 102 |
+
sql = "select key, value from task"
|
| 103 |
+
cursor.execute(sql)
|
| 104 |
+
results = cursor.fetchall()
|
| 105 |
+
obj = {"task": {}, "messages": messages}
|
| 106 |
+
|
| 107 |
+
for key, value in results:
|
| 108 |
+
obj["task"][key] = value
|
| 109 |
+
|
| 110 |
+
return input_path, obj
|
src/elbench/basic_education_runtime/cli/export/exporter/label_studio_.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
from typing import Dict, Any, List
|
| 3 |
+
|
| 4 |
+
from elbench.basic_education_runtime.cli.export.exporter.json_ import aexport_json
|
| 5 |
+
|
| 6 |
+
async def aexport_label_studio(input_path: Path) -> Dict[str, Any]:
|
| 7 |
+
_, json_obj = await aexport_json(input_path)
|
| 8 |
+
msgs = []
|
| 9 |
+
for msg in json_obj["messages"]:
|
| 10 |
+
msgs.append({
|
| 11 |
+
"role": msg["role"],
|
| 12 |
+
"content": msg["content"]
|
| 13 |
+
})
|
| 14 |
+
return {
|
| 15 |
+
"data": {
|
| 16 |
+
"messages": msgs,
|
| 17 |
+
"task_id": input_path.stem
|
| 18 |
+
}
|
| 19 |
+
}
|
src/elbench/basic_education_runtime/cli/export/json_.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json as jsonmodule
|
| 2 |
+
import asyncio
|
| 3 |
+
from tqdm.asyncio import tqdm
|
| 4 |
+
|
| 5 |
+
import click
|
| 6 |
+
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def export_json_logic():
|
| 13 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 14 |
+
|
| 15 |
+
input = CONFIG.globals.memory.path
|
| 16 |
+
output = input
|
| 17 |
+
|
| 18 |
+
dbfiles = []
|
| 19 |
+
files = input.iterdir()
|
| 20 |
+
for file in files:
|
| 21 |
+
if file.suffix == ".db":
|
| 22 |
+
dbfiles.append(file.absolute())
|
| 23 |
+
|
| 24 |
+
from elbench.basic_education_runtime.cli.export.exporter.json_ import aexport_json
|
| 25 |
+
|
| 26 |
+
tasks = []
|
| 27 |
+
for dbfile in dbfiles:
|
| 28 |
+
task = aexport_json(dbfile)
|
| 29 |
+
tasks.append(task)
|
| 30 |
+
|
| 31 |
+
result = asyncio.run(tqdm.gather(*tasks))
|
| 32 |
+
for input_path, obj in result:
|
| 33 |
+
output_path = output / f"{input_path.stem}.json"
|
| 34 |
+
with open(output_path, "w", encoding="utf-8") as f:
|
| 35 |
+
jsonmodule.dump(obj, f, ensure_ascii=False, indent=4)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@click.command(help="Export chat databases to JSON format")
|
| 39 |
+
@click.option(
|
| 40 |
+
"--config", default="config.yaml", help="Directory containing chat databases"
|
| 41 |
+
)
|
| 42 |
+
@click.option("--debug", default=False, help="Debug Mode", is_flag=True)
|
| 43 |
+
def json(config: str, debug: bool):
|
| 44 |
+
set_debug(debug)
|
| 45 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 46 |
+
|
| 47 |
+
path = Path(config)
|
| 48 |
+
load_conf(path)
|
| 49 |
+
export_json_logic()
|
| 50 |
+
|
src/elbench/basic_education_runtime/cli/export/label_studio.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from tqdm.asyncio import tqdm
|
| 4 |
+
from typing import Dict, Any
|
| 5 |
+
|
| 6 |
+
import json as jsonmodule
|
| 7 |
+
import asyncio
|
| 8 |
+
from tqdm.asyncio import tqdm
|
| 9 |
+
|
| 10 |
+
import click
|
| 11 |
+
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def export_label_studio_logic():
|
| 18 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 19 |
+
|
| 20 |
+
input = CONFIG.globals.memory.path
|
| 21 |
+
output = input
|
| 22 |
+
|
| 23 |
+
dbfiles = []
|
| 24 |
+
files = input.iterdir()
|
| 25 |
+
for file in files:
|
| 26 |
+
if file.suffix == ".db":
|
| 27 |
+
dbfiles.append(file.absolute())
|
| 28 |
+
|
| 29 |
+
from elbench.basic_education_runtime.cli.export.exporter.label_studio_ import aexport_label_studio
|
| 30 |
+
from elbench.basic_education_runtime.cli.export.const.label_studio import generate_label_studio_interface
|
| 31 |
+
|
| 32 |
+
if CONFIG.evaluation is None:
|
| 33 |
+
raise ValueError("Evaluation瀛楁鏈厤缃?)
|
| 34 |
+
|
| 35 |
+
tasks = []
|
| 36 |
+
for dbfile in dbfiles:
|
| 37 |
+
task = aexport_label_studio(dbfile)
|
| 38 |
+
tasks.append(task)
|
| 39 |
+
|
| 40 |
+
results = asyncio.run(tqdm.gather(*tasks))
|
| 41 |
+
with open(output / f"label_studio.json", "w", encoding="utf-8") as f:
|
| 42 |
+
jsonmodule.dump(results, f, ensure_ascii=False, indent=4)
|
| 43 |
+
|
| 44 |
+
template = generate_label_studio_interface(CONFIG.evaluation.format)
|
| 45 |
+
with open(output / "label_studio.txt", "w", encoding="utf-8") as f:
|
| 46 |
+
f.write(template)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@click.command(help="Export chat databases to Label Studio Data format")
|
| 51 |
+
@click.option(
|
| 52 |
+
"--config", default="config.yaml", help="Directory containing chat databases"
|
| 53 |
+
)
|
| 54 |
+
@click.option("--debug", default=False, help="Debug Mode", is_flag=True)
|
| 55 |
+
def label_studio(config: str, debug: bool):
|
| 56 |
+
set_debug(debug)
|
| 57 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 58 |
+
|
| 59 |
+
path = Path(config)
|
| 60 |
+
load_conf(path)
|
| 61 |
+
export_label_studio_logic()
|
src/elbench/basic_education_runtime/cli/generate/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 5 |
+
|
| 6 |
+
# set_debug(True)
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@click.command("generate", help="Generate conversions for all tasks")
|
| 10 |
+
@click.option("--config", default="config.yaml", help="Path to the configuration file.")
|
| 11 |
+
@click.option("--debug", default=False, help="Debug Mode", is_flag=True)
|
| 12 |
+
def generate(config: str, debug: bool):
|
| 13 |
+
set_debug(debug)
|
| 14 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 15 |
+
|
| 16 |
+
path = Path(config)
|
| 17 |
+
load_conf(path)
|
| 18 |
+
generate_logic()
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def generate_logic():
|
| 22 |
+
from elbench.basic_education_runtime.run import run
|
| 23 |
+
import asyncio
|
| 24 |
+
|
| 25 |
+
asyncio.run(run())
|
| 26 |
+
|
src/elbench/basic_education_runtime/cli/main.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
|
| 3 |
+
from elbench.basic_education_runtime.langchain_compat import set_debug
|
| 4 |
+
|
| 5 |
+
from elbench.basic_education_runtime.cli.generate import generate, generate_logic
|
| 6 |
+
from elbench.basic_education_runtime.cli.eval import eval, eval_logic
|
| 7 |
+
from elbench.basic_education_runtime.cli.visualize import visualize
|
| 8 |
+
from elbench.basic_education_runtime.cli.draw import draw
|
| 9 |
+
from elbench.basic_education_runtime.cli.export import export
|
| 10 |
+
from elbench.basic_education_runtime.cli.export.json_ import export_json_logic
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@click.command(
|
| 14 |
+
help="Run the pipeline to generate, export JSON files, and evaluate the results."
|
| 15 |
+
)
|
| 16 |
+
@click.option(
|
| 17 |
+
"--config", type=click.Path(exists=True), help="Path to the configuration file"
|
| 18 |
+
)
|
| 19 |
+
@click.option("--debug", is_flag=True, help="Enable debug mode")
|
| 20 |
+
def pipeline(config, debug=False):
|
| 21 |
+
set_debug(debug)
|
| 22 |
+
from elbench.basic_education_runtime.config import load_conf
|
| 23 |
+
|
| 24 |
+
load_conf(config)
|
| 25 |
+
generate_logic()
|
| 26 |
+
export_json_logic()
|
| 27 |
+
eval_logic(avg=True)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@click.group()
|
| 31 |
+
def main():
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
main.add_command(generate)
|
| 36 |
+
main.add_command(export)
|
| 37 |
+
main.add_command(eval)
|
| 38 |
+
main.add_command(pipeline)
|
| 39 |
+
|
| 40 |
+
main.add_command(visualize)
|
| 41 |
+
|
| 42 |
+
main.add_command(draw)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
if __name__ == "__main__":
|
| 46 |
+
main()
|
src/elbench/basic_education_runtime/cli/visualize/__init__.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import click
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@click.command(help="Visualize the results in all CSV file in the specified directory.")
|
| 7 |
+
@click.argument(
|
| 8 |
+
"input_dir",
|
| 9 |
+
type=click.Path(exists=True),
|
| 10 |
+
)
|
| 11 |
+
@click.option(
|
| 12 |
+
"--x-rotation",
|
| 13 |
+
type=int,
|
| 14 |
+
default=30,
|
| 15 |
+
)
|
| 16 |
+
def visualize(input_dir: str, x_rotation: int):
|
| 17 |
+
visualize_logic(input_dir, x_rotation)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def visualize_logic(input_dir: str, x_rotation: int):
|
| 21 |
+
color_palette = [
|
| 22 |
+
"#1f77b4",
|
| 23 |
+
"#aec7e8",
|
| 24 |
+
"#ff7f0e",
|
| 25 |
+
"#ffbb78",
|
| 26 |
+
"#2ca02c",
|
| 27 |
+
"#98df8a",
|
| 28 |
+
"#d62728",
|
| 29 |
+
"#ff9896",
|
| 30 |
+
"#9467bd",
|
| 31 |
+
"#c5b0d5",
|
| 32 |
+
"#8c564b",
|
| 33 |
+
"#c49c94",
|
| 34 |
+
"#e377c2",
|
| 35 |
+
"#f7b6d2",
|
| 36 |
+
"#7f7f7f",
|
| 37 |
+
"#c7c7c7",
|
| 38 |
+
"#bcbd22",
|
| 39 |
+
"#dbdb8d",
|
| 40 |
+
"#17becf",
|
| 41 |
+
"#9edae5",
|
| 42 |
+
]
|
| 43 |
+
import pandas as pd
|
| 44 |
+
import matplotlib.pyplot as plt
|
| 45 |
+
import numpy as np
|
| 46 |
+
from matplotlib import font_manager
|
| 47 |
+
|
| 48 |
+
from importlib.resources import files
|
| 49 |
+
|
| 50 |
+
font_path = files("elbench.basic_education_runtime.assets.fonts").joinpath("sarasa-mono-sc-regular.ttf")
|
| 51 |
+
font_path = str(font_path)
|
| 52 |
+
|
| 53 |
+
font_manager.fontManager.addfont(font_path)
|
| 54 |
+
plt.rcParams["font.sans-serif"] = "Sarasa Mono SC"
|
| 55 |
+
plt.rcParams["axes.unicode_minus"] = False # 瑙e喅璐熷彿鏄剧ず闂
|
| 56 |
+
|
| 57 |
+
input_path = Path(input_dir)
|
| 58 |
+
csvs = input_path.rglob("*.csv")
|
| 59 |
+
|
| 60 |
+
task_name = ""
|
| 61 |
+
keys = []
|
| 62 |
+
models = []
|
| 63 |
+
values = {}
|
| 64 |
+
|
| 65 |
+
for csv in csvs:
|
| 66 |
+
stem_split = csv.stem.rsplit("_", 1)
|
| 67 |
+
if task_name == "":
|
| 68 |
+
task_name = stem_split[0]
|
| 69 |
+
elif task_name != stem_split[0]:
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"Multiple task names found in CSV files. {task_name} and {stem_split[0]} are different."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
model = stem_split[1]
|
| 75 |
+
models.append(model)
|
| 76 |
+
|
| 77 |
+
data = pd.read_csv(csv)
|
| 78 |
+
data = data.drop(columns=["task_id", "avg"])
|
| 79 |
+
data = data.iloc[-1].to_dict()
|
| 80 |
+
|
| 81 |
+
if not keys:
|
| 82 |
+
keys = list(data.keys())
|
| 83 |
+
for k in keys:
|
| 84 |
+
values[k] = []
|
| 85 |
+
elif keys != list(data.keys()):
|
| 86 |
+
raise ValueError(
|
| 87 |
+
f"Data keys do not match across CSV files. [{','.join(keys)}] and [{','.join(data.keys())}] are different."
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
for k in keys:
|
| 91 |
+
values[k].append(data[k])
|
| 92 |
+
|
| 93 |
+
# 鏋勫缓 DataFrame
|
| 94 |
+
df_dict = {"": models}
|
| 95 |
+
for k in keys:
|
| 96 |
+
df_dict[k] = values[k]
|
| 97 |
+
|
| 98 |
+
df = pd.DataFrame(df_dict)
|
| 99 |
+
|
| 100 |
+
ncol = min(len(keys), 5)
|
| 101 |
+
|
| 102 |
+
# ==== 鉁?鑷€傚簲鐢诲竷瀹藉害 ====
|
| 103 |
+
fig_width = max(8, len(df) * 0.8, ncol * 2) # 姣忎釜妯″瀷 0.8 鑻卞锛屾渶灏忓搴︿负 8
|
| 104 |
+
fig, ax = plt.subplots(figsize=(fig_width, 6))
|
| 105 |
+
|
| 106 |
+
df.set_index("").plot(kind="bar", stacked=True, ax=ax, color=color_palette)
|
| 107 |
+
ax.set_xticklabels(df[""], rotation=x_rotation)
|
| 108 |
+
ax.set_title(f"{task_name}")
|
| 109 |
+
|
| 110 |
+
# 鉁?璁剧疆鍥句緥浣嶇疆鍒板浘琛ㄤ笅鏂癸紝鎵撴暎涓哄鍒?
|
| 111 |
+
ax.legend(
|
| 112 |
+
loc="lower center",
|
| 113 |
+
bbox_to_anchor=(0.5, 1.1),
|
| 114 |
+
ncol=ncol,
|
| 115 |
+
frameon=False,
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
plt.tight_layout()
|
| 119 |
+
plt.savefig(input_path / f"stack_{task_name}.png", dpi=300)
|
| 120 |
+
|
| 121 |
+
# ==== 鉁?闆疯揪鍥?====
|
| 122 |
+
# 鍑嗗鏁版嵁
|
| 123 |
+
num_vars = len(keys)
|
| 124 |
+
angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()
|
| 125 |
+
angles += angles[:1] # 闂悎鍥惧舰
|
| 126 |
+
|
| 127 |
+
# 璁$畻鎵€鏈夋暟鍊肩殑鏈€灏忓€?
|
| 128 |
+
all_scores = [values[k] for k in keys]
|
| 129 |
+
min_value = min([min(score_list) for score_list in all_scores])
|
| 130 |
+
|
| 131 |
+
# 璁剧疆闆疯揪鍥剧殑鏈€灏忓€煎師鐐?
|
| 132 |
+
min_value -= 1 # 鏈€灏忓€煎噺鍘?1
|
| 133 |
+
|
| 134 |
+
# 缁樺埗闆疯揪鍥?
|
| 135 |
+
fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(polar=True))
|
| 136 |
+
|
| 137 |
+
for idx, model in enumerate(models):
|
| 138 |
+
scores = [values[k][idx] for k in keys]
|
| 139 |
+
scores += scores[:1] # 闂悎鍥惧舰
|
| 140 |
+
ax.plot(
|
| 141 |
+
angles,
|
| 142 |
+
scores,
|
| 143 |
+
label=model,
|
| 144 |
+
linewidth=2,
|
| 145 |
+
color=color_palette[idx % len(color_palette)],
|
| 146 |
+
)
|
| 147 |
+
ax.fill(angles, scores, alpha=0.1)
|
| 148 |
+
|
| 149 |
+
# 璋冩暣闆疯揪鍥剧殑鍗婂緞鑼冨洿锛岀‘淇濅粠 (min_value - 1) 寮€濮?
|
| 150 |
+
ax.set_ylim(min_value, max([max(score_list) for score_list in all_scores]) + 1)
|
| 151 |
+
|
| 152 |
+
# 璁剧疆鏍囩鍜屾牱寮?
|
| 153 |
+
ax.set_thetagrids(np.degrees(angles[:-1]), keys) # type: ignore
|
| 154 |
+
ax.set_title(f"{task_name}", size=16)
|
| 155 |
+
ax.legend(loc="upper right", bbox_to_anchor=(1.3, 1.1))
|
| 156 |
+
plt.tight_layout()
|
| 157 |
+
plt.savefig(input_path / f"radar_{task_name}.png", dpi=300)
|
| 158 |
+
|
src/elbench/basic_education_runtime/config.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from elbench.basic_education_runtime.entity import BasicEducationRuntimeConfig
|
| 2 |
+
from elbench.basic_education_runtime.utils import extract
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Dict, Any
|
| 5 |
+
|
| 6 |
+
import yaml
|
| 7 |
+
|
| 8 |
+
CONFIG: BasicEducationRuntimeConfig
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def load_conf(path: Path):
|
| 12 |
+
if isinstance(path, str):
|
| 13 |
+
path = Path(path)
|
| 14 |
+
if not path.exists():
|
| 15 |
+
return
|
| 16 |
+
global CONFIG
|
| 17 |
+
data: Dict[str, Dict[str, Any]] = {}
|
| 18 |
+
try:
|
| 19 |
+
with open(path, "r", encoding="utf8") as f:
|
| 20 |
+
t = f.read()
|
| 21 |
+
for d in yaml.safe_load_all(t):
|
| 22 |
+
data = d
|
| 23 |
+
# 缂栫爜閿欒
|
| 24 |
+
except UnicodeDecodeError:
|
| 25 |
+
with open(path, "r", encoding="gbk") as f:
|
| 26 |
+
t = f.read()
|
| 27 |
+
for d in yaml.safe_load_all(t):
|
| 28 |
+
data = d
|
| 29 |
+
|
| 30 |
+
n_data = {}
|
| 31 |
+
for k in data.keys():
|
| 32 |
+
c = extract(data, k)
|
| 33 |
+
n_data[k] = c
|
| 34 |
+
if n_data["globals"].get("memory", {}).get("path", None) is None:
|
| 35 |
+
if "memory" not in n_data["globals"]:
|
| 36 |
+
n_data["globals"]["memory"] = {}
|
| 37 |
+
n_data["globals"]["memory"]["path"] = path.parent / path.stem
|
| 38 |
+
CONFIG = BasicEducationRuntimeConfig(**n_data)
|
| 39 |
+
if CONFIG.evaluation is not None:
|
| 40 |
+
if CONFIG.evaluation.name is None:
|
| 41 |
+
CONFIG.evaluation.name = path.stem
|
| 42 |
+
|
src/elbench/basic_education_runtime/directions.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from typing import Dict, Any, Tuple, Optional
|
| 3 |
+
from uuid import uuid4
|
| 4 |
+
from langgraph.graph.state import CompiledStateGraph
|
| 5 |
+
from langgraph.graph import StateGraph, START, END
|
| 6 |
+
from langgraph.prebuilt.chat_agent_executor import AgentState
|
| 7 |
+
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
| 8 |
+
|
| 9 |
+
import aiosqlite
|
| 10 |
+
|
| 11 |
+
from elbench.basic_education_runtime.entity import AgentConfig
|
| 12 |
+
from elbench.basic_education_runtime.router import * # noqa: F403
|
| 13 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def add_node_to_graph(graph: StateGraph, node_id: str, node_instance: Any) -> None:
|
| 17 |
+
if node_id not in graph.nodes:
|
| 18 |
+
graph.add_node(node_id, node_instance)
|
| 19 |
+
else:
|
| 20 |
+
pass
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
async def apply_agent_direction_from_dict(
|
| 24 |
+
agent_map: Dict[str, Tuple[CompiledStateGraph, AgentConfig]],
|
| 25 |
+
memory_id: Optional[str] = None,
|
| 26 |
+
task: Optional[Dict[str, str]] = None,
|
| 27 |
+
) -> Tuple[CompiledStateGraph, str]:
|
| 28 |
+
if memory_id is None:
|
| 29 |
+
memory_id = str(uuid4())
|
| 30 |
+
directions = CONFIG.directions
|
| 31 |
+
graph = StateGraph(AgentState)
|
| 32 |
+
for node_id, node_instance in agent_map.items():
|
| 33 |
+
graph.add_node(node_id, node_instance[0])
|
| 34 |
+
|
| 35 |
+
for direction in directions:
|
| 36 |
+
start_node, end_node = (i.strip() for i in direction.split("->"))
|
| 37 |
+
if start_node == "START":
|
| 38 |
+
start_node = START
|
| 39 |
+
if end_node.startswith("router:"):
|
| 40 |
+
function_call = end_node[len("router:") :]
|
| 41 |
+
route, path_map = eval(function_call)
|
| 42 |
+
end_node = end_node.replace(":", "_")
|
| 43 |
+
graph.add_conditional_edges(start_node, route, path_map)
|
| 44 |
+
continue
|
| 45 |
+
# add_node_to_graph(graph, end_node, route)
|
| 46 |
+
elif end_node == "END":
|
| 47 |
+
end_node = END
|
| 48 |
+
else:
|
| 49 |
+
agent_config = agent_map[end_node][1]
|
| 50 |
+
pregel_instance = agent_map[end_node][0]
|
| 51 |
+
if not pregel_instance or not agent_config:
|
| 52 |
+
raise ValueError(f"Invalid configuration for {end_node}.")
|
| 53 |
+
graph.add_edge(start_node, end_node)
|
| 54 |
+
CONFIG.globals.memory.path.mkdir(parents=True, exist_ok=True)
|
| 55 |
+
path = CONFIG.globals.memory.path / f"{memory_id}.db"
|
| 56 |
+
# conn = sqlite3.connect(f"{memory_id}.db", check_same_thread=False)
|
| 57 |
+
if task is not None:
|
| 58 |
+
async with aiosqlite.connect(path) as conn:
|
| 59 |
+
sql = "create table task (key TEXT, value TEXT)"
|
| 60 |
+
await conn.execute(sql)
|
| 61 |
+
sql = "insert into task (key, value) values (?, ?)"
|
| 62 |
+
for key, value in task.items():
|
| 63 |
+
await conn.execute(sql, (key, value))
|
| 64 |
+
await conn.commit()
|
| 65 |
+
# await conn.close()
|
| 66 |
+
conn = aiosqlite.connect(path, check_same_thread=False)
|
| 67 |
+
if not hasattr(conn, "is_alive"):
|
| 68 |
+
conn.is_alive = lambda: bool( # type: ignore[attr-defined]
|
| 69 |
+
getattr(conn, "_running", False) and getattr(conn, "_connection", None) is not None
|
| 70 |
+
)
|
| 71 |
+
CONFIG.context.conns.append(conn)
|
| 72 |
+
|
| 73 |
+
memory = AsyncSqliteSaver(conn)
|
| 74 |
+
return graph.compile(checkpointer=memory), memory_id
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
apply_agent_direction = apply_agent_direction_from_dict
|
| 78 |
+
|
src/elbench/basic_education_runtime/entity.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import re
|
| 3 |
+
from typing import Dict, Any, Literal, Optional, List, Annotated, Tuple, Final
|
| 4 |
+
from pydantic import BaseModel, ConfigDict, Field, create_model
|
| 5 |
+
from typing_extensions import TypedDict
|
| 6 |
+
from langgraph.graph.message import add_messages
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from aiosqlite import Connection
|
| 9 |
+
from polyfactory.factories.pydantic_factory import ModelFactory
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
# Common
|
| 14 |
+
class State(TypedDict):
|
| 15 |
+
messages: Annotated[list, add_messages]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# Memory
|
| 19 |
+
class Memory(BaseModel):
|
| 20 |
+
path: Path = Path(".")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# RetryConfig
|
| 24 |
+
class RetryConfig(BaseModel):
|
| 25 |
+
attempt: int = 3
|
| 26 |
+
interval: int = 3
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# Global
|
| 30 |
+
class GlobalConfig(BaseModel):
|
| 31 |
+
concurrency: int = 8
|
| 32 |
+
recursion_limit: int = 25
|
| 33 |
+
memory: Memory = Memory()
|
| 34 |
+
retry: RetryConfig = RetryConfig()
|
| 35 |
+
model_call_timeout_seconds: int = 600
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# Model
|
| 39 |
+
class ModelConfig(BaseModel):
|
| 40 |
+
api_base: Optional[str]
|
| 41 |
+
api_key: Optional[str]
|
| 42 |
+
kargs: Optional[Dict[str, Any]] = None
|
| 43 |
+
model: Optional[str]
|
| 44 |
+
type: str = "openai"
|
| 45 |
+
# Which part of the run this model plays (target, student, judge); used only for spending limits.
|
| 46 |
+
role: Optional[str] = None
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# Agent
|
| 50 |
+
class Prompt(BaseModel):
|
| 51 |
+
role: Optional[str]
|
| 52 |
+
content: str
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class SwitchConfig(BaseModel):
|
| 56 |
+
swap_user_assistant: bool = True
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class AgentMemoryConfig(BaseModel):
|
| 60 |
+
enable: bool = True
|
| 61 |
+
id: Optional[str] = None
|
| 62 |
+
keep_turns: int = 3
|
| 63 |
+
# when_switch: SwitchConfig = SwitchConfig()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class AgentConfig(BaseModel):
|
| 67 |
+
model: str
|
| 68 |
+
prompt: Final[List[Prompt]]
|
| 69 |
+
memory: AgentMemoryConfig = AgentMemoryConfig(enable=True)
|
| 70 |
+
|
| 71 |
+
checkpointer: Optional[Any] = None
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# Task
|
| 75 |
+
class TaskConfig(BaseModel):
|
| 76 |
+
start_prompt: Optional[Prompt] = None
|
| 77 |
+
variables: List[Dict[str, str]] = []
|
| 78 |
+
|
| 79 |
+
def model_post_init(self, __context):
|
| 80 |
+
object.__setattr__(self, "_frozen_start_prompt", True)
|
| 81 |
+
|
| 82 |
+
def __setattr__(self, name, value):
|
| 83 |
+
if getattr(self, f"_frozen_{name}", False) and name == "start_prompt":
|
| 84 |
+
raise AttributeError(f"{name} is const and cannot be modified")
|
| 85 |
+
super().__setattr__(name, value)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# Runtime Context
|
| 89 |
+
class BasicEducationRuntimeContext(BaseModel):
|
| 90 |
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
| 91 |
+
conns: List[Connection] = []
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# ExportFormat
|
| 95 |
+
class ExportFormat(BaseModel):
|
| 96 |
+
task: Dict[str, str]
|
| 97 |
+
messages: List[Prompt] = []
|
| 98 |
+
|
| 99 |
+
@staticmethod
|
| 100 |
+
def from_json_file(file_path: Path | str) -> "ExportFormat":
|
| 101 |
+
with open(file_path, "r", encoding="utf-8") as f:
|
| 102 |
+
data = json.load(f)
|
| 103 |
+
return ExportFormat(**data)
|
| 104 |
+
|
| 105 |
+
def message_function(self, function_call: str) -> str:
|
| 106 |
+
if function_call == "as_dialog()":
|
| 107 |
+
strings = []
|
| 108 |
+
for message in self.messages:
|
| 109 |
+
if message.role is None:
|
| 110 |
+
continue
|
| 111 |
+
strings.append(f"{message.role}: {message.content}")
|
| 112 |
+
return "\n".join(strings)
|
| 113 |
+
else:
|
| 114 |
+
raise ValueError(f"Unsupported function call: {function_call}")
|
| 115 |
+
|
| 116 |
+
def replace_template(self, template: str) -> str:
|
| 117 |
+
"""
|
| 118 |
+
将输入的模板字符串中的占位符替换为实际值
|
| 119 |
+
目前支持的占位符有:
|
| 120 |
+
task.xxxx 代表task的某个字段
|
| 121 |
+
messages.as_dialog() 代表对话形式的messages
|
| 122 |
+
|
| 123 |
+
messages.as_dialog() 形式如下:
|
| 124 |
+
teacher: xxxxxxx
|
| 125 |
+
student: xxxxxxx
|
| 126 |
+
"""
|
| 127 |
+
# Only runtime placeholders should be substituted. Model outputs may
|
| 128 |
+
# legitimately contain braces such as "{1}", which must remain literal.
|
| 129 |
+
placeholders = re.findall(r"\{(?:task\.[^{}]+|messages\.[^{}]+)\}", template)
|
| 130 |
+
for placeholder in placeholders:
|
| 131 |
+
# 获取占位符的名称,即{}中的内容
|
| 132 |
+
placeholder_name = placeholder.strip("{}")
|
| 133 |
+
# 获取占位符对应的值,根据占位符的名称从task或messages中获取
|
| 134 |
+
if placeholder_name.startswith("task."):
|
| 135 |
+
field_name = placeholder_name.split(".")[1]
|
| 136 |
+
value = self.task[field_name]
|
| 137 |
+
elif placeholder_name.startswith("messages."):
|
| 138 |
+
message_name = placeholder_name.split(".")[1]
|
| 139 |
+
# 如果是函数调用形式
|
| 140 |
+
if "(" in message_name:
|
| 141 |
+
value = self.message_function(message_name)
|
| 142 |
+
else:
|
| 143 |
+
raise Exception(f"Invalid message name: {message_name}")
|
| 144 |
+
else:
|
| 145 |
+
raise Exception(f"Invalid placeholder name: {placeholder_name}")
|
| 146 |
+
# 将占位符替换为对应的值
|
| 147 |
+
template = template.replace(placeholder, str(value))
|
| 148 |
+
return template
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# FormatField
|
| 152 |
+
class FormatField(BaseModel):
|
| 153 |
+
field: str
|
| 154 |
+
type: Literal["str", "int", "float", "bool", "dict"]
|
| 155 |
+
description: str
|
| 156 |
+
items: List["FormatField"] = []
|
| 157 |
+
# min: float | int | None = None
|
| 158 |
+
max: float | int | None = None
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# Evaluation
|
| 162 |
+
class EvalConfig(BaseModel):
|
| 163 |
+
model: str
|
| 164 |
+
name: Optional[str]
|
| 165 |
+
prompt: List[Prompt]
|
| 166 |
+
format: List[FormatField]
|
| 167 |
+
format_mode: Literal["tool", "prompt"] = "tool"
|
| 168 |
+
|
| 169 |
+
def format_to_json_schema(self) -> str:
|
| 170 |
+
model = self.format_to_pydantic()
|
| 171 |
+
json_schema = model.model_json_schema()
|
| 172 |
+
return json.dumps(json_schema, ensure_ascii=False)
|
| 173 |
+
|
| 174 |
+
def format_to_json_example(self) -> str:
|
| 175 |
+
mmodel: type[BaseModel] = self.format_to_pydantic()
|
| 176 |
+
|
| 177 |
+
class MMF(ModelFactory):
|
| 178 |
+
__model__ = mmodel
|
| 179 |
+
|
| 180 |
+
return MMF().build().json()
|
| 181 |
+
|
| 182 |
+
def get_prompts(self) -> Tuple[str, List[Prompt]]:
|
| 183 |
+
"""获取系统提示和其他提示词"""
|
| 184 |
+
system_prompt = ""
|
| 185 |
+
other_prompt: List[Prompt] = []
|
| 186 |
+
for p in self.prompt:
|
| 187 |
+
if p.role == "system":
|
| 188 |
+
if system_prompt != "":
|
| 189 |
+
raise ValueError("存在多个系统提示")
|
| 190 |
+
system_prompt = p.content
|
| 191 |
+
else:
|
| 192 |
+
other_prompt.append(p)
|
| 193 |
+
return system_prompt, other_prompt
|
| 194 |
+
|
| 195 |
+
def format_to_pydantic(self) -> type[BaseModel]:
|
| 196 |
+
def field_type_from_format(f: FormatField) -> tuple:
|
| 197 |
+
"""将FormatField转成pydantic字段元组(类型,Field信息)"""
|
| 198 |
+
python_type_map = {
|
| 199 |
+
"str": str,
|
| 200 |
+
"int": int,
|
| 201 |
+
"float": float,
|
| 202 |
+
"bool": bool,
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
if f.type == "dict":
|
| 206 |
+
nested_model = build_model_from_format(f.items, f.field)
|
| 207 |
+
return nested_model, Field(..., description=f.description)
|
| 208 |
+
if f.type == "int" or f.type == "float":
|
| 209 |
+
py_type = python_type_map[f.type]
|
| 210 |
+
# if f.min is None:
|
| 211 |
+
# if f.max is None:
|
| 212 |
+
# return py_type, Field(..., description=f.description)
|
| 213 |
+
# else:
|
| 214 |
+
# return py_type, Field(..., le=f.max, description=f.description)
|
| 215 |
+
# else:
|
| 216 |
+
# if f.max is None:
|
| 217 |
+
# return py_type, Field(..., ge=f.min, description=f.description)
|
| 218 |
+
# else:
|
| 219 |
+
# return py_type, Field(..., ge=f.min, le=f.max, description=f.description)
|
| 220 |
+
|
| 221 |
+
if f.max is None:
|
| 222 |
+
return py_type, Field(..., description=f.description)
|
| 223 |
+
else:
|
| 224 |
+
return py_type, Field(..., le=f.max, description=f.description)
|
| 225 |
+
else:
|
| 226 |
+
py_type = python_type_map[f.type]
|
| 227 |
+
return py_type, Field(..., description=f.description)
|
| 228 |
+
|
| 229 |
+
def build_model_from_format(
|
| 230 |
+
fields: List[FormatField], model_name: str = "DynamicModel"
|
| 231 |
+
) -> type[BaseModel]:
|
| 232 |
+
annotations = {}
|
| 233 |
+
for f in fields:
|
| 234 |
+
annotations[f.field] = field_type_from_format(f)
|
| 235 |
+
return create_model(model_name, **annotations)
|
| 236 |
+
|
| 237 |
+
return build_model_from_format(self.format, "GeneratedModel")
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
# Runtime
|
| 241 |
+
class BasicEducationRuntimeConfig(BaseModel):
|
| 242 |
+
globals: GlobalConfig
|
| 243 |
+
models: Dict[str, ModelConfig]
|
| 244 |
+
agents: Dict[str, AgentConfig]
|
| 245 |
+
directions: List[str]
|
| 246 |
+
tasks: TaskConfig
|
| 247 |
+
evaluation: Optional[EvalConfig] = None
|
| 248 |
+
|
| 249 |
+
context: BasicEducationRuntimeContext = BasicEducationRuntimeContext(conns=[])
|
src/elbench/basic_education_runtime/evaluation.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import asyncio
|
| 3 |
+
from typing import Any, Dict
|
| 4 |
+
from langchain_core.tools import tool, BaseTool
|
| 5 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 6 |
+
from langgraph.prebuilt import create_react_agent
|
| 7 |
+
from langchain.chat_models.base import BaseChatModel
|
| 8 |
+
from elbench.basic_education_runtime.entity import ExportFormat
|
| 9 |
+
from pydantic import BaseModel
|
| 10 |
+
from tenacity import retry, stop_after_attempt, wait_fixed
|
| 11 |
+
import json
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def generate_evaluation_tool() -> BaseTool:
|
| 15 |
+
"""
|
| 16 |
+
Generates a tool for evaluating the basic education runtime outputs.
|
| 17 |
+
This function is a placeholder and should be implemented with actual logic.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
if CONFIG.evaluation is None:
|
| 21 |
+
raise ValueError("Evaluation configuration not found.")
|
| 22 |
+
|
| 23 |
+
@tool(
|
| 24 |
+
name_or_callable="save_result_to_database",
|
| 25 |
+
description="Save the evaluation results to a database.",
|
| 26 |
+
return_direct=True,
|
| 27 |
+
args_schema=CONFIG.evaluation.format_to_pydantic(),
|
| 28 |
+
)
|
| 29 |
+
def save_to_db(**kwargs):
|
| 30 |
+
"""
|
| 31 |
+
Save the evaluation results to a database.
|
| 32 |
+
"""
|
| 33 |
+
for k, v in kwargs.items():
|
| 34 |
+
if issubclass(v.__class__, BaseModel):
|
| 35 |
+
v = v.model_dump()
|
| 36 |
+
kwargs[k] = v
|
| 37 |
+
return kwargs
|
| 38 |
+
|
| 39 |
+
return save_to_db
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _build_evaluation_messages(
|
| 43 |
+
*,
|
| 44 |
+
system_prompt: str,
|
| 45 |
+
other_prompts: list[Any],
|
| 46 |
+
exported_result: ExportFormat,
|
| 47 |
+
) -> list[dict[str, str]]:
|
| 48 |
+
messages: list[dict[str, str]] = []
|
| 49 |
+
for op in other_prompts:
|
| 50 |
+
content = exported_result.replace_template(op.content)
|
| 51 |
+
role = str(op.role or "user").strip() or "user"
|
| 52 |
+
messages.append(
|
| 53 |
+
{
|
| 54 |
+
"role": role,
|
| 55 |
+
"content": content,
|
| 56 |
+
}
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
has_user_query = any(
|
| 60 |
+
item.get("role") == "user" and str(item.get("content", "")).strip()
|
| 61 |
+
for item in messages
|
| 62 |
+
)
|
| 63 |
+
if not has_user_query:
|
| 64 |
+
fallback = exported_result.replace_template(system_prompt).strip()
|
| 65 |
+
if not fallback:
|
| 66 |
+
fallback = "Please complete the evaluation."
|
| 67 |
+
messages.append(
|
| 68 |
+
{
|
| 69 |
+
"role": "user",
|
| 70 |
+
"content": fallback,
|
| 71 |
+
}
|
| 72 |
+
)
|
| 73 |
+
return messages
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@retry(
|
| 77 |
+
stop=stop_after_attempt(CONFIG.globals.retry.attempt),
|
| 78 |
+
wait=wait_fixed(CONFIG.globals.retry.interval),
|
| 79 |
+
)
|
| 80 |
+
async def evaluate(
|
| 81 |
+
model: BaseChatModel, exported_result: ExportFormat
|
| 82 |
+
) -> Dict[str, Any]:
|
| 83 |
+
assert CONFIG.evaluation
|
| 84 |
+
system_prompt, other_prompt = CONFIG.evaluation.get_prompts()
|
| 85 |
+
system_prompt = exported_result.replace_template(system_prompt)
|
| 86 |
+
ops = _build_evaluation_messages(
|
| 87 |
+
system_prompt=system_prompt,
|
| 88 |
+
other_prompts=other_prompt,
|
| 89 |
+
exported_result=exported_result,
|
| 90 |
+
)
|
| 91 |
+
mc = getattr(model, "model_config", None)
|
| 92 |
+
if getattr(mc, "type", None) == "system_one":
|
| 93 |
+
from elbench.judges.system_one_judge import evaluate_basic_education_system_one
|
| 94 |
+
|
| 95 |
+
return await evaluate_basic_education_system_one(
|
| 96 |
+
api_base=mc.api_base,
|
| 97 |
+
api_key=mc.api_key,
|
| 98 |
+
model_name=str(mc.model or "liquid/d1-ultra"),
|
| 99 |
+
timeout_seconds=float(CONFIG.globals.model_call_timeout_seconds or 60.0),
|
| 100 |
+
retry_attempts=int(CONFIG.globals.retry.attempt or 1),
|
| 101 |
+
system_prompt=system_prompt,
|
| 102 |
+
messages=ops,
|
| 103 |
+
format_fields=list(CONFIG.evaluation.format),
|
| 104 |
+
)
|
| 105 |
+
if CONFIG.evaluation.format_mode == "tool":
|
| 106 |
+
tools = [generate_evaluation_tool()]
|
| 107 |
+
|
| 108 |
+
# other_prompt = exported_result.replace_template(other_prompt)
|
| 109 |
+
agent = create_react_agent(
|
| 110 |
+
model=model.bind_tools(
|
| 111 |
+
tools,
|
| 112 |
+
tool_choice={
|
| 113 |
+
"type": "function",
|
| 114 |
+
"function": {"name": "save_result_to_database"},
|
| 115 |
+
}, # type: ignore
|
| 116 |
+
# tool_choice="required",
|
| 117 |
+
),
|
| 118 |
+
tools=tools,
|
| 119 |
+
# model=model,
|
| 120 |
+
prompt=system_prompt
|
| 121 |
+
+ "\n\nPlease call the save_result_to_database tool to store the evaluation result.",
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
a = await asyncio.wait_for(
|
| 125 |
+
agent.ainvoke({"messages": ops}),
|
| 126 |
+
timeout=CONFIG.globals.model_call_timeout_seconds,
|
| 127 |
+
)
|
| 128 |
+
data = json.loads(a["messages"][-1].content)
|
| 129 |
+
return data
|
| 130 |
+
elif CONFIG.evaluation.format_mode == "prompt":
|
| 131 |
+
format_example = CONFIG.evaluation.format_to_json_example()
|
| 132 |
+
agent = create_react_agent(
|
| 133 |
+
model=model,
|
| 134 |
+
tools=[],
|
| 135 |
+
prompt=system_prompt
|
| 136 |
+
+ "\n\nReturn exactly one JSON object wrapped with <START OF EVAL OUTPUT> and <END OF EVAL OUTPUT>."
|
| 137 |
+
+ "\nDo not add markdown fences, explanations, or extra text."
|
| 138 |
+
+ "\nUse the same keys and value types as this example:\n"
|
| 139 |
+
+ "\n<START OF EVAL OUTPUT>\n"
|
| 140 |
+
+ f"{format_example}\n"
|
| 141 |
+
+ "<END OF EVAL OUTPUT>",
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
a = await asyncio.wait_for(
|
| 145 |
+
agent.ainvoke({"messages": ops}),
|
| 146 |
+
timeout=CONFIG.globals.model_call_timeout_seconds,
|
| 147 |
+
)
|
| 148 |
+
response: str = a["messages"][-1].content
|
| 149 |
+
|
| 150 |
+
# Fuck Gemini
|
| 151 |
+
response = response.replace("\\<", "<")
|
| 152 |
+
response = response.replace("\\>", ">")
|
| 153 |
+
|
| 154 |
+
# print(response)
|
| 155 |
+
|
| 156 |
+
# 姝e垯琛ㄨ揪寮忓尮閰?START OUTPUT>鍜?END OUTPUT>
|
| 157 |
+
match = re.findall(
|
| 158 |
+
r"<START OF EVAL OUTPUT>(.*)<END OF EVAL OUTPUT>", response, re.DOTALL
|
| 159 |
+
)
|
| 160 |
+
if len(match) > 0:
|
| 161 |
+
text = match[-1]
|
| 162 |
+
# 濡傛灉text琚玚``鍖呭洿锛屽幓鎺?
|
| 163 |
+
text = text.strip().strip("```").strip("json")
|
| 164 |
+
# print(text, "#############")
|
| 165 |
+
return json.loads(text)
|
| 166 |
+
else:
|
| 167 |
+
raise ValueError("Output does not contain <START OUTPUT> and <END OUTPUT>")
|
| 168 |
+
else:
|
| 169 |
+
raise ValueError(f"Invalid format mode: {CONFIG.evaluation.format_mode}")
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
if __name__ == "__main__":
|
| 173 |
+
import asyncio
|
| 174 |
+
|
| 175 |
+
async def main():
|
| 176 |
+
from elbench.basic_education_runtime.model import init_chat_model_from_dict
|
| 177 |
+
# from langchain.globals import set_debug
|
| 178 |
+
|
| 179 |
+
# set_debug(True)
|
| 180 |
+
|
| 181 |
+
ef = ExportFormat.from_json_file(
|
| 182 |
+
"/tmp/basic_education_runtime/sample.json"
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
model = init_chat_model_from_dict(CONFIG.models["4o_teacher"])
|
| 186 |
+
a = await evaluate(model, ef)
|
| 187 |
+
print(a)
|
| 188 |
+
|
| 189 |
+
asyncio.run(main())
|
| 190 |
+
|
src/elbench/basic_education_runtime/langchain_compat.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compatibility helpers for LangChain version differences."""
|
| 2 |
+
|
| 3 |
+
try:
|
| 4 |
+
from langchain.globals import set_debug
|
| 5 |
+
except ModuleNotFoundError:
|
| 6 |
+
try:
|
| 7 |
+
from langchain_core.globals import set_debug
|
| 8 |
+
except ModuleNotFoundError:
|
| 9 |
+
def set_debug(_: bool) -> None:
|
| 10 |
+
"""Fallback when LangChain debug globals are unavailable."""
|
| 11 |
+
|
| 12 |
+
return None
|
src/elbench/basic_education_runtime/model.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any, Dict
|
| 4 |
+
|
| 5 |
+
from langchain.chat_models import init_chat_model
|
| 6 |
+
from langchain.chat_models.base import BaseChatModel
|
| 7 |
+
from langchain_core.messages import AIMessage
|
| 8 |
+
|
| 9 |
+
from elbench.basic_education_runtime.config import CONFIG
|
| 10 |
+
from elbench.basic_education_runtime.entity import ModelConfig
|
| 11 |
+
from elbench.basic_education_runtime.spend_callback import SpendCallback
|
| 12 |
+
from elbench.utils import spend
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class MockRuntimeChatModel:
|
| 16 |
+
def __init__(self, model_config: ModelConfig) -> None:
|
| 17 |
+
self.model_config = model_config
|
| 18 |
+
|
| 19 |
+
async def ainvoke(self, messages: list[Any]) -> AIMessage:
|
| 20 |
+
prefix = ""
|
| 21 |
+
if self.model_config.kargs:
|
| 22 |
+
prefix = str(self.model_config.kargs.get("prefix", "")).strip()
|
| 23 |
+
|
| 24 |
+
last_content = ""
|
| 25 |
+
for item in reversed(messages):
|
| 26 |
+
content = getattr(item, "content", None)
|
| 27 |
+
if isinstance(content, str) and content.strip():
|
| 28 |
+
last_content = content.strip()
|
| 29 |
+
break
|
| 30 |
+
if isinstance(item, dict):
|
| 31 |
+
dict_content = item.get("content")
|
| 32 |
+
if isinstance(dict_content, str) and dict_content.strip():
|
| 33 |
+
last_content = dict_content.strip()
|
| 34 |
+
break
|
| 35 |
+
|
| 36 |
+
summary = last_content[:120]
|
| 37 |
+
content = "占位回答:我们一步一步来。<end>"
|
| 38 |
+
if summary:
|
| 39 |
+
content = f"{prefix} 占位回答:已收到问题。{summary}\n我们一步一步来。<end>".strip()
|
| 40 |
+
elif prefix:
|
| 41 |
+
content = f"{prefix} {content}".strip()
|
| 42 |
+
return AIMessage(content=content)
|
| 43 |
+
|
| 44 |
+
def bind_tools(self, tools: list[Any], tool_choice: Any = None) -> "MockRuntimeChatModel":
|
| 45 |
+
return self
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class SystemOneRuntimeModel:
|
| 49 |
+
def __init__(self, model_config: ModelConfig) -> None:
|
| 50 |
+
self.model_config = model_config
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def init_chat_model_from_dict(mc: ModelConfig) -> BaseChatModel:
|
| 54 |
+
if mc.type == "mock":
|
| 55 |
+
return MockRuntimeChatModel(mc) # type: ignore[return-value]
|
| 56 |
+
if mc.type == "system_one":
|
| 57 |
+
return SystemOneRuntimeModel(mc) # type: ignore[return-value]
|
| 58 |
+
if mc.kargs is not None:
|
| 59 |
+
llm = init_chat_model(
|
| 60 |
+
model=f"{mc.type}:{mc.model}",
|
| 61 |
+
api_key=mc.api_key,
|
| 62 |
+
base_url=mc.api_base,
|
| 63 |
+
**mc.kargs,
|
| 64 |
+
)
|
| 65 |
+
else:
|
| 66 |
+
llm = init_chat_model(
|
| 67 |
+
model=f"{mc.type}:{mc.model}",
|
| 68 |
+
api_key=mc.api_key,
|
| 69 |
+
base_url=mc.api_base,
|
| 70 |
+
)
|
| 71 |
+
if mc.role and spend.active_ledger() is not None:
|
| 72 |
+
llm.callbacks = [*(llm.callbacks or []), SpendCallback(mc.role)]
|
| 73 |
+
return llm
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def init_model_map_from_dict() -> Dict[str, BaseChatModel]:
|
| 77 |
+
cfg = CONFIG.models
|
| 78 |
+
result = {}
|
| 79 |
+
for k, v in cfg.items():
|
| 80 |
+
result[k] = init_chat_model_from_dict(v)
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
init_model_map = init_model_map_from_dict
|
| 85 |
+
|
| 86 |
+
if __name__ == "__main__":
|
| 87 |
+
a = init_model_map_from_dict()
|
| 88 |
+
print(a)
|