adelevett commited on
Commit
5db30ac
·
verified ·
1 Parent(s): 621d357

Deploy ELBench-live dashboard

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. .gitignore +24 -0
  3. README.md +153 -7
  4. README.zh-CN.md +100 -0
  5. configs/app.yaml +27 -0
  6. configs/basic_education.yaml +71 -0
  7. configs/field_mappings.yaml +98 -0
  8. configs/file_registry.yaml +241 -0
  9. configs/gpt54_guided_FIXED.yaml +213 -0
  10. configs/gpt54_guided_SMOKE.yaml +207 -0
  11. configs/judges.yaml +35 -0
  12. configs/models.yaml +473 -0
  13. configs/module_registry.yaml +17 -0
  14. configs/providers.yaml +21 -0
  15. dashboard/README.md +57 -0
  16. dashboard/app.py +1405 -0
  17. docs/ARCHITECTURE.md +297 -0
  18. docs/ARCHITECTURE.zh-CN.md +274 -0
  19. docs/DASHBOARD_TARGETS.md +187 -0
  20. docs/RESULT_STORAGE_SPEC.zh-CN.md +148 -0
  21. docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md +298 -0
  22. docs/benchmark_status_2026-05-25.md +97 -0
  23. pyproject.toml +58 -0
  24. requirements.txt +24 -0
  25. scripts/build_stage_report.py +423 -0
  26. scripts/repair_satas_reference_and_rejudge.py +293 -0
  27. scripts/run_benchmark.py +14 -0
  28. src/elbench/__init__.py +1 -0
  29. src/elbench/basic_education_runtime/__init__.py +2 -0
  30. src/elbench/basic_education_runtime/agent.py +85 -0
  31. src/elbench/basic_education_runtime/assets/__init__.py +1 -0
  32. src/elbench/basic_education_runtime/assets/fonts/__init__.py +1 -0
  33. src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf +3 -0
  34. src/elbench/basic_education_runtime/cli/draw/__init__.py +35 -0
  35. src/elbench/basic_education_runtime/cli/eval/__init__.py +147 -0
  36. src/elbench/basic_education_runtime/cli/export/__init__.py +19 -0
  37. src/elbench/basic_education_runtime/cli/export/const/label_studio.py +89 -0
  38. src/elbench/basic_education_runtime/cli/export/exporter/json_.py +110 -0
  39. src/elbench/basic_education_runtime/cli/export/exporter/label_studio_.py +19 -0
  40. src/elbench/basic_education_runtime/cli/export/json_.py +50 -0
  41. src/elbench/basic_education_runtime/cli/export/label_studio.py +61 -0
  42. src/elbench/basic_education_runtime/cli/generate/__init__.py +26 -0
  43. src/elbench/basic_education_runtime/cli/main.py +46 -0
  44. src/elbench/basic_education_runtime/cli/visualize/__init__.py +158 -0
  45. src/elbench/basic_education_runtime/config.py +42 -0
  46. src/elbench/basic_education_runtime/directions.py +78 -0
  47. src/elbench/basic_education_runtime/entity.py +249 -0
  48. src/elbench/basic_education_runtime/evaluation.py +190 -0
  49. src/elbench/basic_education_runtime/langchain_compat.py +12 -0
  50. src/elbench/basic_education_runtime/model.py +88 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ .DS_Store
3
+ *.pyc
4
+ *.pyo
5
+ *.pyd
6
+ .pytest_cache/
7
+ .mypy_cache/
8
+ .ruff_cache/
9
+ .DS_Store
10
+ pytest-cache-files-*/
11
+ .conda/
12
+ .venv/
13
+ venv/
14
+ dist/
15
+ build/
16
+ *.egg-info/
17
+ outputs/
18
+ key.txt
19
+ tmp/
20
+ tmp_test_artifacts/
21
+ results/
22
+
23
+ # Local secrets
24
+ .env
README.md CHANGED
@@ -1,13 +1,159 @@
1
  ---
2
- title: ELBench Live
3
- emoji: 🌖
4
- colorFrom: red
5
  colorTo: indigo
6
  sdk: gradio
7
- sdk_version: 6.30.0
8
- python_version: '3.12'
9
- app_file: app.py
 
10
  pinned: false
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: ELBench-live
3
+ emoji: "📚"
4
+ colorFrom: blue
5
  colorTo: indigo
6
  sdk: gradio
7
+ sdk_version: 6.4.0
8
+ python_version: "3.12"
9
+ app_file: dashboard/app.py
10
+ suggested_hardware: cpu-basic
11
  pinned: false
12
  ---
13
 
14
+ # ELBench / ELBench-live
15
+
16
+ [![English](https://img.shields.io/badge/Language-English-blue.svg)](./README.md)
17
+ [![简体中文](https://img.shields.io/badge/Language-%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87-red.svg)](./README.zh-CN.md)
18
+
19
+ 📄 [**Paper (arXiv:2608.09548)**](https://arxiv.org/abs/2608.09548) · 💻 [**GitHub (ZeroLoss-Lab/ELBench)**](https://github.com/ZeroLoss-Lab/ELBench) · 🏆 [**AI for Education Leaderboard**](https://benchmarks.ai-for-education.org/) · 🗂️ [**Benchmark Dataset**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) · 📊 [**Results Dataset**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)
20
+
21
+ **ELBench** is a multi-dimensional benchmark for education-facing large language
22
+ models. It evaluates a model across four modules — general capability, safety
23
+ and trustworthiness, basic education, and high-level educational cultivation —
24
+ under one consistent, config-driven evaluation framework.
25
+
26
+ The benchmark covers **2,939 items** across the four modules and is built so a
27
+ single aggregate score never hides where a model actually succeeds or fails.
28
+
29
+ ## Leaderboard
30
+
31
+ Overall scores are reported on a 0–100 scale with a bootstrap 95% confidence
32
+ interval over items (10,000 resamples). The top models are statistically close:
33
+ the spread among the first six is under two points, and several share rank 4.
34
+
35
+ | Rank | Model | Overall [95% CI] | General | Safety | Basic Ed. | High-Level |
36
+ |:----:|:------|:----------------:|:-------:|:------:|:---------:|:----------:|
37
+ | 1 | DeepSeek-V4-Flash | **83.7** [82, 85] | 88.5 | 89.7 | 84.9 | 71.5 |
38
+ | 2 | Gemini-3.5-Flash | **83.4** [82, 84] | 93.4 | 70.2 | 94.6 | 75.3 |
39
+ | 3 | Doubao-Seed-2.0-Pro | **83.2** [82, 84] | 86.6 | 83.0 | 89.7 | 73.5 |
40
+ | 4 | Claude-Opus-4.8 | **83.1** [82, 84] | 91.9 | 78.5 | 92.7 | 69.5 |
41
+ | 4 | GPT-5.4 | **83.1** [82, 84] | 86.4 | 76.6 | 94.4 | 75.1 |
42
+ | 4 | DeepSeek-V4-Pro | **83.1** [82, 84] | 88.2 | 86.1 | 88.5 | 69.5 |
43
+ | 7 | GLM-5.1 | **81.7** [80, 84] | 83.2 | 89.5 | 79.2 | 75.0 |
44
+ | 8 | Safe-InnoSpark | **77.0** [76, 78] | 68.0 | 87.6 | 87.3 | 65.2 |
45
+ | 9 | InnoSpark-235B | **76.4** [75, 78] | 74.2 | 78.0 | 87.5 | 65.9 |
46
+
47
+ Per-module and per-task leaderboards, confidence intervals, and the
48
+ judge-reliability audit are released as the
49
+ [ZeroLoss-Lab/ELBench-results](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)
50
+ dataset on HuggingFace.
51
+
52
+ ## Modules
53
+
54
+ | Module | Items | What it measures |
55
+ |:-------|:-----:|:-----------------|
56
+ | General Capability (`通用模型`) | 894 | Knowledge, reasoning, instruction following, and math (MMLU-Pro, C-Eval, IFEval, MATH-500, AIME samples) |
57
+ | Safety & Trustworthiness (`安全可信`) | 1,000 | Harmful-request refusal, helpfulness on benign requests, and adversarial robustness |
58
+ | Basic Education (`基本教育`) | 45 | Multi-turn teaching: knowledge explanation, situated question design, cross-subject lesson plans, guided problem solving |
59
+ | High-Level Educational Cultivation (`高阶育人`) | 1,000 | Open-ended cultivation of higher-order educational goals |
60
+
61
+ Objective tasks are graded by rule or reference matching; subjective and
62
+ open-ended tasks use `LLM-as-a-Judge` with a validated judge panel. The
63
+ judge-agreement audit (quadratic weighted Cohen's κ against a human gold set)
64
+ is in the [results dataset](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results).
65
+
66
+ ## Installation
67
+
68
+ ```bash
69
+ pip install -e . # core framework
70
+ pip install -e .[basic-education] # + Basic Education multi-turn runtime
71
+ pip install -e .[basic-education,dev] # + contributor tooling
72
+ ```
73
+
74
+ ## Quick Start
75
+
76
+ ```bash
77
+ # Inspect the registered benchmark files
78
+ python scripts/run_benchmark.py inspect
79
+
80
+ # Local smoke test against the mock provider (no API keys needed)
81
+ python scripts/run_benchmark.py run \
82
+ --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
83
+ ```
84
+
85
+ After an editable install the `elbench` entrypoint is also available:
86
+
87
+ ```bash
88
+ elbench inspect
89
+ elbench run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
90
+ ```
91
+
92
+ To evaluate a real model, add it to `configs/models.yaml` and run a module:
93
+
94
+ ```bash
95
+ python scripts/run_benchmark.py run \
96
+ --model-id <your_model_id> --module 基本教育 --run-id basic-education-run
97
+ ```
98
+
99
+ `基本教育` (Basic Education) requires a real API-backed model config; the
100
+ `mock.*` models are for single-turn pipeline smoke checks only.
101
+
102
+ ## Hosted runner
103
+
104
+ `dashboard/app.py` is a Gradio app for a visitor who brings an OpenRouter key.
105
+ It calls `BenchmarkRunner` and `JudgeRouter`. Prompts, retries, and scores are
106
+ not reimplemented in the UI. The estimate states the cost range and the ways a
107
+ subset, a `max_samples` prefix, or an unlisted model diverges from a published
108
+ run. The YAML block at the top of this file is the Hugging Face Space config.
109
+ CPU Basic has no hourly price, but creating the Gradio Space requires a paid plan.
110
+ Details are in [dashboard/README.md](dashboard/README.md).
111
+
112
+ ```bash
113
+ pip install -e ".[basic-education,dashboard]"
114
+ python dashboard/app.py
115
+ ```
116
+
117
+ ## Data on HuggingFace
118
+
119
+ ELBench is released as two HuggingFace datasets:
120
+
121
+ - [**ZeroLoss-Lab/ELBench**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) — the benchmark items (2,939 across the four modules).
122
+ - [**ZeroLoss-Lab/ELBench-results**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) — the full evaluation results, including per-sample detail.
123
+
124
+ This repository holds the evaluation framework and code only; the benchmark
125
+ items and results live in the two datasets above. Run artifacts
126
+ (`outputs/raw_responses/`, `judged_results/`, `logs/`) are runtime-only and not
127
+ committed.
128
+
129
+ ## Documentation
130
+
131
+ - [Architecture](docs/ARCHITECTURE.md) — registry, provider abstraction,
132
+ execution model, and how to extend the framework
133
+ - [Team maintenance guide](docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md) — what each
134
+ directory is for and what should or should not be committed
135
+
136
+ ## Tests
137
+
138
+ ```bash
139
+ python -m unittest \
140
+ tests.test_basic_education_bridge tests.test_registry_and_loaders \
141
+ tests.test_judges tests.test_response_format_retry
142
+ ```
143
+
144
+ ## 🔬 Citation
145
+
146
+ If you use this benchmark in your research, please cite:
147
+
148
+ ```bibtex
149
+ @article{jiang2026elbench,
150
+ title={ELBench: A Multi-Dimensional Benchmark for Education-Facing Large Language Models},
151
+ author={Jiang, Yilin and Zhu, Xiaorong and Tan, Fei and Zhang, Zicheng and Huang, Kaiyi and Yu, Yang and Fei, Zexuan and Luo, Yiming and Li, Keqian and Hao, Hao and others},
152
+ journal={arXiv preprint arXiv:2608.09548},
153
+ year={2026}
154
+ }
155
+ ```
156
+
157
+ ## License
158
+
159
+ Add your preferred license before publishing.
README.zh-CN.md ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench
2
+
3
+ [![English](https://img.shields.io/badge/Language-English-blue.svg)](./README.md)
4
+ [![简体中文](https://img.shields.io/badge/Language-%E7%AE%80%E4%BD%93%E4%B8%AD%E6%96%87-red.svg)](./README.zh-CN.md)
5
+
6
+ **ELBench** 是一个面向教育大模型的多维度评测基准。它从四个模块评估模型——通用能力、安全可信、基本教育、高阶育人——并在统一的、配置驱动的评测框架下运行。
7
+
8
+ 基准共覆盖 **2939 道题**,横跨四个模块;设计上让单一总分无法掩盖模型在各模块上的真实强弱。
9
+
10
+ ## 排行榜
11
+
12
+ 总分采用 0–100 量纲,并给出基于条目级 bootstrap(10000 次重采样)的 95% 置信区间。头部模型相当接近:前六名分差不到两分,多个模型并列第 4。
13
+
14
+ | 排名 | 模型 | 总分 [95% CI] | 通用 | 安全 | 基本教育 | 高阶育人 |
15
+ |:----:|:-----|:------------:|:----:|:----:|:--------:|:--------:|
16
+ | 1 | DeepSeek-V4-Flash | **83.7** [82, 85] | 88.5 | 89.7 | 84.9 | 71.5 |
17
+ | 2 | Gemini-3.5-Flash | **83.4** [82, 84] | 93.4 | 70.2 | 94.6 | 75.3 |
18
+ | 3 | Doubao-Seed-2.0-Pro | **83.2** [82, 84] | 86.6 | 83.0 | 89.7 | 73.5 |
19
+ | 4 | Claude-Opus-4.8 | **83.1** [82, 84] | 91.9 | 78.5 | 92.7 | 69.5 |
20
+ | 4 | GPT-5.4 | **83.1** [82, 84] | 86.4 | 76.6 | 94.4 | 75.1 |
21
+ | 4 | DeepSeek-V4-Pro | **83.1** [82, 84] | 88.2 | 86.1 | 88.5 | 69.5 |
22
+ | 7 | GLM-5.1 | **81.7** [80, 84] | 83.2 | 89.5 | 79.2 | 75.0 |
23
+ | 8 | Safe-InnoSpark | **77.0** [76, 78] | 68.0 | 87.6 | 87.3 | 65.2 |
24
+ | 9 | InnoSpark-235B | **76.4** [75, 78] | 74.2 | 78.0 | 87.5 | 65.9 |
25
+
26
+ 各模块、各任务的明细榜、置信区间以及评分可靠性审计,以 HuggingFace 数据集
27
+ [ZeroLoss-Lab/ELBench-results](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) 发布。
28
+
29
+ ## 模块
30
+
31
+ | 模块 | 题量 | 评测内容 |
32
+ |:-----|:----:|:---------|
33
+ | 通用能力(`通用模型`) | 894 | 知识、推理、指令遵循与数学(MMLU-Pro、C-Eval、IFEval、MATH-500、AIME 采样) |
34
+ | 安全可信(`安全可信`) | 1000 | 有害请求拒答、良性请求的有用性、对抗鲁棒性 |
35
+ | 基本教育(`基本教育`) | 45 | 多轮教学:知识点讲解、情景化出题、跨学科教案、引导式讲题 |
36
+ | 高阶育人(`高阶育人`) | 1000 | 高阶育人目标的开放式培养 |
37
+
38
+ 客观题由规则或参考答案匹配判分;主观和开放式任务采用 `LLM-as-a-Judge`,并配有经过验证的评分模型面板。评分一致性审计(对人工金标的二次加权 Cohen κ)见
39
+ [结果数据集](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results)。
40
+
41
+ ## 安装
42
+
43
+ ```bash
44
+ pip install -e . # 核心框架
45
+ pip install -e .[basic-education] # + 基本教育多轮运行时
46
+ pip install -e .[basic-education,dev] # + 贡献者工具链
47
+ ```
48
+
49
+ ## 快速开始
50
+
51
+ ```bash
52
+ # 查看已注册的 benchmark 文件
53
+ python scripts/run_benchmark.py inspect
54
+
55
+ # 用 mock provider 跑本地冒烟测试(无需 API key)
56
+ python scripts/run_benchmark.py run \
57
+ --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
58
+ ```
59
+
60
+ editable 安装后也可直接用 `elbench` 入口:
61
+
62
+ ```bash
63
+ elbench inspect
64
+ elbench run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
65
+ ```
66
+
67
+ 评测真实模型时,先在 `configs/models.yaml` 里登记模型,再按模块运行:
68
+
69
+ ```bash
70
+ python scripts/run_benchmark.py run \
71
+ --model-id <你的模型ID> --module 基本教育 --run-id basic-education-run
72
+ ```
73
+
74
+ `基本教育` 需要真实的 API 模型配置;`mock.*` 模型仅用于单轮流水线冒烟检查。
75
+
76
+ ## HuggingFace 数据集
77
+
78
+ ELBench 以两个 HuggingFace 数据集发布:
79
+
80
+ - [**ZeroLoss-Lab/ELBench**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench) —— 评测题目集(四模块共 2939 题)。
81
+ - [**ZeroLoss-Lab/ELBench-results**](https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results) —— 完整评测结果,含逐样本明细。
82
+
83
+ 本仓库只保留评测框架与代码;评测题目与结果都在上面两个数据集中。原始单次运行产物(`outputs/raw_responses/`、`judged_results/`、`logs/`)只在运行时存在,不进版本库。
84
+
85
+ ## 文档
86
+
87
+ - [架构说明](docs/ARCHITECTURE.zh-CN.md):注册表、provider 抽象、执行模型,以及如何扩展框架
88
+ - [团队维护指南](docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md):各目录的用途,以及哪些该提交、哪些不该提交
89
+
90
+ ## 测试
91
+
92
+ ```bash
93
+ python -m unittest \
94
+ tests.test_basic_education_bridge tests.test_registry_and_loaders \
95
+ tests.test_judges tests.test_response_format_retry
96
+ ```
97
+
98
+ ## 许可证
99
+
100
+ 发布前请补充你选择的许可证。
configs/app.yaml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ project_name: ELBench
2
+ data_root: data/benchmark_root
3
+ output_root: outputs
4
+ default_run:
5
+ max_concurrency: 100
6
+ checkpoint_interval: 1
7
+ log_every_n: 50
8
+ judge_enabled: true
9
+ summary_enabled: true
10
+ resume: true
11
+ retry:
12
+ max_attempts: 3
13
+ initial_delay_seconds: 1.0
14
+ max_delay_seconds: 20.0
15
+ backoff_multiplier: 2.0
16
+ jitter_ratio: 0.2
17
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
18
+ provider_rate_limits:
19
+ default:
20
+ max_concurrency: 100
21
+ benchmark_modules:
22
+ active:
23
+ - 安全可信
24
+ - 高阶育人
25
+ - 通用模型
26
+ - 基本教育
27
+ reserved: []
configs/basic_education.yaml ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ enabled: true
2
+ module_name: 基本教育
3
+ runtime_python: python
4
+ runtime_cli_module: elbench.basic_education_runtime.cli.main
5
+ command_timeout_seconds: 14400
6
+ continue_on_error: true
7
+ workspace_root: outputs/basic_education
8
+ test_endpoint_model_id: qwen-test
9
+ evaluator_model_id: null
10
+ default_pass_threshold: 3.0
11
+ scenarios:
12
+ - scenario_id: knowledge_point_explanation
13
+ source_file: knowledge.yaml
14
+ subset: 知识点讲解
15
+ task: basic_knowledge_point_explanation
16
+ template_path: 基本教育/知识点讲解/knowledge.yaml
17
+ expected_tasks: 10
18
+ multi_turn: false
19
+ dimension: 知识点讲解
20
+ target_model_keys: [4o_teacher]
21
+ test_endpoint_model_keys: []
22
+ judge_model_keys: [evaler]
23
+ pass_threshold: 3.0
24
+ metadata:
25
+ topic: basic_education
26
+ scene: knowledge
27
+ - scenario_id: contextualized_question_generation
28
+ source_file: question.yaml
29
+ subset: 情景化出题
30
+ task: basic_contextualized_question_generation
31
+ template_path: 基本教育/情景化出题/question.yaml
32
+ expected_tasks: 10
33
+ multi_turn: false
34
+ dimension: 情景化出题
35
+ target_model_keys: [test_model]
36
+ test_endpoint_model_keys: []
37
+ judge_model_keys: [judge_model]
38
+ pass_threshold: 3.0
39
+ metadata:
40
+ topic: basic_education
41
+ scene: question
42
+ - scenario_id: interdisciplinary_lesson_plan_generation
43
+ source_file: cross.yaml
44
+ subset: 跨学科教案生成
45
+ task: basic_interdisciplinary_lesson_plan_generation
46
+ template_path: 基本教育/跨学科教案生成/cross.yaml
47
+ expected_tasks: 10
48
+ multi_turn: false
49
+ dimension: 跨学科教案生成
50
+ target_model_keys: [test_model]
51
+ test_endpoint_model_keys: []
52
+ judge_model_keys: [judge_model]
53
+ pass_threshold: 3.0
54
+ metadata:
55
+ topic: basic_education
56
+ scene: cross
57
+ - scenario_id: guided_problem_solving_teaching
58
+ source_file: config_guided_task.yaml
59
+ subset: 引导式讲题
60
+ task: basic_guided_problem_solving_teaching
61
+ template_path: 基本教育/引导式讲题/config_guided_task.yaml
62
+ expected_tasks: 15
63
+ multi_turn: true
64
+ dimension: 引导式讲题
65
+ target_model_keys: [teacher]
66
+ test_endpoint_model_keys: [stu]
67
+ judge_model_keys: [eval]
68
+ pass_threshold: 3.0
69
+ metadata:
70
+ topic: basic_education
71
+ scene: guided
configs/field_mappings.yaml ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mappings:
2
+ safety_refusal_jsonl:
3
+ sample_id_fields: []
4
+ prompt_fields: [question]
5
+ dimension_fields: []
6
+ reference_fields: [label]
7
+ metadata_fields: [label, subcategory]
8
+ safety_guidance_jsonl:
9
+ sample_id_fields: []
10
+ prompt_fields: [question]
11
+ dimension_fields: []
12
+ reference_fields: [question_type, category]
13
+ metadata_fields: [question_type, category, subcategory]
14
+ safety_answer_jsonl:
15
+ sample_id_fields: []
16
+ prompt_fields: [question]
17
+ dimension_fields: []
18
+ reference_fields: [label]
19
+ metadata_fields: [label, subcategory]
20
+ teaching_harm_xlsx:
21
+ sample_id_fields: [ID]
22
+ dimension_fields: []
23
+ reference_fields: [Answer]
24
+ metadata_fields: [Question_Chinese, Question_English, Answer]
25
+ prompt_template: |
26
+ [Question_Chinese]
27
+ {{ Question_Chinese }}
28
+
29
+ [Question_English]
30
+ {{ Question_English }}
31
+ adversarial_safety_xlsx:
32
+ sample_id_fields: [ID]
33
+ dimension_fields: []
34
+ reference_fields: []
35
+ metadata_fields:
36
+ [Teacher_Prompt_EN, Student_Statement_EN, Teacher_Prompt_CN, Student_Statement_CN]
37
+ prompt_template: |
38
+ [Teacher_Prompt_CN]
39
+ {{ Teacher_Prompt_CN }}
40
+
41
+ [Student_Statement_CN]
42
+ {{ Student_Statement_CN }}
43
+
44
+ [Teacher_Prompt_EN]
45
+ {{ Teacher_Prompt_EN }}
46
+
47
+ [Student_Statement_EN]
48
+ {{ Student_Statement_EN }}
49
+ highlevel_edu_jsonl:
50
+ sample_id_fields: []
51
+ prompt_fields: [question]
52
+ dimension_fields: []
53
+ dimension_resolver: highlevel_edu
54
+ reference_fields: [Score, Scoring Details, Personalized Feedback]
55
+ metadata_fields:
56
+ [Subject, Level, Question Type, Question, Student's Answer, Score, Scoring Details, Personalized Feedback, _scene]
57
+ highlevel_omni_jsonl:
58
+ sample_id_fields: [id]
59
+ prompt_fields: [question]
60
+ dimension_fields: [field]
61
+ reference_fields: [answer, reasonableness]
62
+ metadata_fields: [type, field, answer, reasonableness]
63
+ basic_education_yaml:
64
+ sample_id_fields: []
65
+ prompt_fields: []
66
+ dimension_fields: []
67
+ reference_fields: []
68
+ metadata_fields: []
69
+ mmlu_pro_jsonl:
70
+ sample_id_fields: [id]
71
+ prompt_fields: []
72
+ dimension_fields: [subset_key]
73
+ reference_fields: [target]
74
+ metadata_fields: [metadata, choices, subset_key]
75
+ ceval_jsonl:
76
+ sample_id_fields: [id]
77
+ prompt_fields: []
78
+ dimension_fields: [metadata.subject]
79
+ reference_fields: [target]
80
+ metadata_fields: [metadata, choices, subset_key]
81
+ ifeval_jsonl:
82
+ sample_id_fields: [id]
83
+ prompt_fields: []
84
+ dimension_fields: []
85
+ reference_fields: [target]
86
+ metadata_fields: [metadata, instruction_id_list, kwargs, key, prompt]
87
+ math_500_jsonl:
88
+ sample_id_fields: [id]
89
+ prompt_fields: []
90
+ dimension_fields: [subset_key]
91
+ reference_fields: [target]
92
+ metadata_fields: [metadata, subset_key]
93
+ aime_jsonl:
94
+ sample_id_fields: [id]
95
+ prompt_fields: []
96
+ dimension_fields: []
97
+ reference_fields: [target]
98
+ metadata_fields: [metadata, subset_key]
configs/file_registry.yaml ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ files:
2
+ - canonical_name: 安全拒答.jsonl
3
+ module: 安全可信
4
+ subset: 通用-应拒答
5
+ file_type: jsonl
6
+ task: safety_refusal
7
+ loader_name: jsonl
8
+ mapping_key: safety_refusal_jsonl
9
+ path_hints:
10
+ - 安全可信/通用-应拒答/安全拒答.jsonl
11
+ - 安全拒答.jsonl
12
+ filename_patterns:
13
+ - 安全拒答.jsonl
14
+ aliases: []
15
+ - canonical_name: 安全引导.jsonl
16
+ module: 安全可信
17
+ subset: 通用-引导回答
18
+ file_type: jsonl
19
+ task: safety_guidance
20
+ loader_name: jsonl
21
+ mapping_key: safety_guidance_jsonl
22
+ path_hints:
23
+ - 安全可信/通用-引导回答/安全引导.jsonl
24
+ - 安全引导.jsonl
25
+ filename_patterns:
26
+ - 安全引导.jsonl
27
+ aliases: []
28
+ - canonical_name: 安全回答.jsonl
29
+ module: 安全可信
30
+ subset: 通用-应回答
31
+ file_type: jsonl
32
+ task: safety_answer
33
+ loader_name: jsonl
34
+ mapping_key: safety_answer_jsonl
35
+ path_hints:
36
+ - 安全可信/通用-应回答/安全回答.jsonl
37
+ - 安全回答.jsonl
38
+ filename_patterns:
39
+ - 安全回答.jsonl
40
+ aliases: []
41
+ - canonical_name: SATAs.xlsx
42
+ module: 安全可信
43
+ subset: 教育-教学安全
44
+ file_type: xlsx
45
+ task: teaching_harm
46
+ loader_name: xlsx
47
+ mapping_key: teaching_harm_xlsx
48
+ path_hints:
49
+ - 安全可信/教育-教学安全/SATAs.xlsx
50
+ - SATAs.xlsx
51
+ - SATAs(150).xlsx
52
+ filename_patterns:
53
+ - SATAs.xlsx
54
+ - SATAs*.xlsx
55
+ aliases:
56
+ - SATAs(150).xlsx
57
+ - canonical_name: adversarial_prompts.xlsx
58
+ module: 安全可信
59
+ subset: 教育-教学安全
60
+ file_type: xlsx
61
+ task: adversarial_safety
62
+ loader_name: xlsx
63
+ mapping_key: adversarial_safety_xlsx
64
+ path_hints:
65
+ - 安全可信/教育-教学安全/adversarial_prompts.xlsx
66
+ - adversarial_prompts.xlsx
67
+ - adversarial_prompts(100).xlsx
68
+ filename_patterns:
69
+ - adversarial_prompts.xlsx
70
+ - adversarial_prompts*.xlsx
71
+ aliases:
72
+ - adversarial_prompts(100).xlsx
73
+ - canonical_name: 高阶育人-edu.jsonl
74
+ module: 高阶育人
75
+ subset: edu
76
+ file_type: jsonl
77
+ task: highlevel_edu
78
+ loader_name: jsonl
79
+ mapping_key: highlevel_edu_jsonl
80
+ path_hints:
81
+ - 高阶育人/edu/高阶育人-edu.jsonl
82
+ - 高阶育人-edu.jsonl
83
+ filename_patterns:
84
+ - 高阶育人-edu.jsonl
85
+ aliases: []
86
+ - canonical_name: 高阶育人-omni.jsonl
87
+ module: 高阶育人
88
+ subset: omni
89
+ file_type: jsonl
90
+ task: highlevel_omni
91
+ loader_name: jsonl
92
+ mapping_key: highlevel_omni_jsonl
93
+ path_hints:
94
+ - 高阶育人/omni/高阶育人-omni.jsonl
95
+ - 高阶育人-omni.jsonl
96
+ filename_patterns:
97
+ - 高阶育人-omni.jsonl
98
+ aliases: []
99
+ - canonical_name: knowledge.yaml
100
+ module: 基本教育
101
+ subset: 知识点讲解
102
+ file_type: yaml
103
+ task: basic_knowledge_point_explanation
104
+ loader_name: basic_education_yaml
105
+ mapping_key: basic_education_yaml
106
+ path_hints:
107
+ - 基本教育/知识点讲解/knowledge.yaml
108
+ - knowledge.yaml
109
+ filename_patterns:
110
+ - knowledge.yaml
111
+ aliases: []
112
+ - canonical_name: question.yaml
113
+ module: 基本教育
114
+ subset: 情景化出题
115
+ file_type: yaml
116
+ task: basic_contextualized_question_generation
117
+ loader_name: basic_education_yaml
118
+ mapping_key: basic_education_yaml
119
+ path_hints:
120
+ - 基本教育/情景化出题/question.yaml
121
+ - question.yaml
122
+ filename_patterns:
123
+ - question.yaml
124
+ aliases: []
125
+ - canonical_name: cross.yaml
126
+ module: 基本教育
127
+ subset: 跨学科教案生成
128
+ file_type: yaml
129
+ task: basic_interdisciplinary_lesson_plan_generation
130
+ loader_name: basic_education_yaml
131
+ mapping_key: basic_education_yaml
132
+ path_hints:
133
+ - 基本教育/跨学科教案生成/cross.yaml
134
+ - cross.yaml
135
+ filename_patterns:
136
+ - cross.yaml
137
+ aliases: []
138
+ - canonical_name: config_guided_task.yaml
139
+ module: 基本教育
140
+ subset: 引导式讲题
141
+ file_type: yaml
142
+ task: basic_guided_problem_solving_teaching
143
+ loader_name: basic_education_yaml
144
+ mapping_key: basic_education_yaml
145
+ path_hints:
146
+ - 基本教育/引导式讲题/config_guided_task.yaml
147
+ - config_guided_task.yaml
148
+ filename_patterns:
149
+ - config_guided_task.yaml
150
+ aliases: []
151
+ - canonical_name: mmlu_pro_sampled.jsonl
152
+ module: 通用模型
153
+ subset: mmlu_pro
154
+ file_type: jsonl
155
+ task: mmlu_pro
156
+ loader_name: mmlu_pro_jsonl
157
+ mapping_key: mmlu_pro_jsonl
158
+ path_hints:
159
+ - 通用/mmlu_pro_sampled.jsonl
160
+ - mmlu_pro_sampled.jsonl
161
+ filename_patterns:
162
+ - mmlu_pro_sampled.jsonl
163
+ aliases: []
164
+ - canonical_name: ceval_sampled.jsonl
165
+ module: 通用模型
166
+ subset: ceval
167
+ file_type: jsonl
168
+ task: ceval
169
+ loader_name: ceval_jsonl
170
+ mapping_key: ceval_jsonl
171
+ path_hints:
172
+ - 通用/ceval_sampled.jsonl
173
+ - ceval_sampled.jsonl
174
+ filename_patterns:
175
+ - ceval_sampled.jsonl
176
+ aliases: []
177
+ - canonical_name: ifeval_sampled.jsonl
178
+ module: 通用模型
179
+ subset: ifeval
180
+ file_type: jsonl
181
+ task: ifeval
182
+ loader_name: ifeval_jsonl
183
+ mapping_key: ifeval_jsonl
184
+ path_hints:
185
+ - 通用/ifeval_sampled.jsonl
186
+ - ifeval_sampled.jsonl
187
+ filename_patterns:
188
+ - ifeval_sampled.jsonl
189
+ aliases: []
190
+ - canonical_name: math_500_sampled.jsonl
191
+ module: 通用模型
192
+ subset: math_500
193
+ file_type: jsonl
194
+ task: math_500
195
+ loader_name: math_500_jsonl
196
+ mapping_key: math_500_jsonl
197
+ path_hints:
198
+ - 通用/math_500_sampled.jsonl
199
+ - math_500_sampled.jsonl
200
+ filename_patterns:
201
+ - math_500_sampled.jsonl
202
+ aliases: []
203
+ - canonical_name: aime24_sampled.jsonl
204
+ module: 通用模型
205
+ subset: aime24
206
+ file_type: jsonl
207
+ task: aime24
208
+ loader_name: aime_jsonl
209
+ mapping_key: aime_jsonl
210
+ path_hints:
211
+ - 通用/aime24_sampled.jsonl
212
+ - aime24_sampled.jsonl
213
+ filename_patterns:
214
+ - aime24_sampled.jsonl
215
+ aliases: []
216
+ - canonical_name: aime25.jsonl
217
+ module: 通用模型
218
+ subset: aime25
219
+ file_type: jsonl
220
+ task: aime25
221
+ loader_name: aime_jsonl
222
+ mapping_key: aime_jsonl
223
+ path_hints:
224
+ - 通用/aime25.jsonl
225
+ - aime25.jsonl
226
+ filename_patterns:
227
+ - aime25.jsonl
228
+ aliases: []
229
+ - canonical_name: aime26.jsonl
230
+ module: 通用模型
231
+ subset: aime26
232
+ file_type: jsonl
233
+ task: aime26
234
+ loader_name: aime_jsonl
235
+ mapping_key: aime_jsonl
236
+ path_hints:
237
+ - 通用/aime26.jsonl
238
+ - aime26.jsonl
239
+ filename_patterns:
240
+ - aime26.jsonl
241
+ aliases: []
configs/gpt54_guided_FIXED.yaml ADDED
@@ -0,0 +1,213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ globals:
2
+ concurrency: 4
3
+ recursion_limit: 20
4
+ retry:
5
+ attempt: 15
6
+ interval: 3
7
+ memory:
8
+ path: ./outputs/basic_education/campaign-official-gpt-5.4-guided-rerun/gpt-5.4/guided_problem_solving_teaching/memory
9
+ models:
10
+ teacher:
11
+ type: openai
12
+ api_key: sk-ThOC1tDHyquvogfkzNUHFV9545RT0Cg17DnYkXlSO5p4evRd
13
+ api_base: http://35.220.164.252:3888/v1
14
+ model: gpt-5.4
15
+ kargs:
16
+ temperature: 0.0
17
+ max_tokens: 4096
18
+ stu:
19
+ type: openai
20
+ api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
21
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
22
+ model: qwen
23
+ kargs:
24
+ temperature: 0.0
25
+ timeout: 180
26
+ request_timeout: 180
27
+ max_tokens: 1024
28
+ extra_body:
29
+ chat_template_kwargs: &id001
30
+ enable_thinking: false
31
+ served_model_name: qwen3.6-35B-sft-v1
32
+ eval:
33
+ type: openai
34
+ api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
35
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
36
+ model: qwen
37
+ kargs:
38
+ temperature: 0.0
39
+ timeout: 180
40
+ request_timeout: 180
41
+ max_tokens: 1024
42
+ extra_body:
43
+ chat_template_kwargs: *id001
44
+ served_model_name: qwen3.6-35B-sft-v1
45
+ agents:
46
+ teacher:
47
+ model: teacher
48
+ prompt:
49
+ - role: system
50
+ content: '| 你是一位富有耐心和爱心的数学老师,今天你的任务是为这位学生提供一对一的引导式教学。以下是学生的画像信息:{image}。请根据该学生的特点,个性化地引导讲解数学题,在教学过程中自然融入个性化支持、价值观引导以及创造力激发。
51
+
52
+ ---
53
+
54
+ \#\#\# 教学要求
55
+
56
+ 请采用**引导式教学法**帮助学生解决数学问题,具体要求如下:
57
+
58
+ * 以**一问一答**的方式进行教学,每次仅提出**一个引导性问题**,等待学生回答后再继续; * 不要直接给出答案,而应通过启发式提问引导学生自主思考;
59
+ * 根据学生的学习特点,保持耐心,使用**简单清晰的语言**进行表达; * 在讲解过程中加入**积极的鼓励**与**情感支持**,增强学生的自信心;
60
+ * 在适当场合融入**中国传统价值观**,如坚持、尊师重道、勤奋等; * 将复杂问题拆解为学生能理解的**小步骤**; * 尽量联系**生活实际场景**,使数学概念具体、生动。
61
+
62
+ ---
63
+
64
+ \#\#\# 创造力培养
65
+
66
+ * 鼓励学生**从多个角度思考问题**,不拘泥于标准解法; * 引导其发现数学问题的多种可能解决方案; * 当学生尝试自主解题时,给予积极反馈和必要指导;
67
+ * 善于将数学概念与学生兴趣(如**足球等体育活动**)相结合,激发创造性思维; * 设计**开放性问题**,培养学生的发散性思维; * 鼓励学生用自己的语言表达对数学概念的理解;
68
+ * 对学生的创新思路和尝试表示赞赏,即使答案并不完全正确。
69
+
70
+ ---
71
+
72
+ 请开始你的教学引导。请记住:
73
+
74
+ * 一次只提一个问题; * 保持语言简洁、态度耐心、语气鼓励; * 不要提示过多,也**不要直接给出答案**; * 绝对不要替学生回答问题**,你的职责是当一名好老师;
75
+ * 你每次只输出老师要说的内容,**不要同时扮演老师和学生 * 教学结束时,请明确表示“下课”,可以使用 `<end>` 作为结束标志。'
76
+ memory:
77
+ enable: true
78
+ id: null
79
+ keep_turns: 3
80
+ student:
81
+ model: stu
82
+ prompt:
83
+ - role: system
84
+ content: '你将扮演一名{image}的学生, 正在与一位老师进行一对一的数学学习。你的核心任务不是简单地给出正确答案,而是要模拟一个真实学生的学习过程。请严格遵守以下行为准则:一、
85
+ 你的思考与互动方式:展现思考过程: 在回答问题时,尽量“思考出声”。可以说出你的初步想法、计算步骤,即使它们是错误的。例如:“我先试试把这个数代进去看看……”
86
+ 主动提问: 当遇到不理解的概念、模糊的词语或不确定的步骤时,必须向老师提问。例如:“老师,您刚才说的‘互换’是指只换数字,位置不变吗?”允许犯错:
87
+ 不要害怕犯错。你的回答中可以包含常见的计算错误、逻辑卡点或概念混淆,等待老师的纠正。这是你学习过程的一部分。表达情绪和状态: 适度地表达你的学习状态。例如,可以说“啊,这里我有点卡住了”、“哦!我好像明白了!”或者“这个方法真有趣!”
88
+ 二、 你的语言风格:使用符合你学生身份的、自然的、口语化的语言。避免使用过于专业、书面化或总结性的词汇(例如,不要说“该问题的本质是……”)。三、
89
+ 关键禁止事项:禁止一步到位: 绝对不要直接给出完美的、最终的答案和推导。禁止扮演老师: 绝对不要评价老师的教学方法,或者用专家的口吻进行总结。禁止超越认知:
90
+ 不要提出或使用远超当前课题范围的知识点。这是本次一对一辅导所要讲的习题: {question}。'
91
+ directions:
92
+ - START -> teacher
93
+ - teacher -> router:any_keyword_route(keywords=["下课", "这节课就上到这里", "今天的课程结束", "本节课结束",
94
+ "课程到此结束", "同学们,今天这节课就上到这里", "今天的课程就讲到这里", "再见", "祝你学习顺利", "随时联系我", "下次课见", "大家再见",
95
+ "期待下次课", "<end>"], exists_to=END, else_to="student")
96
+ - student -> teacher
97
+ tasks:
98
+ start_prompt:
99
+ role: user
100
+ content: '这是本次一对一辅导所要讲的习题: {question}'
101
+ mode: union
102
+ content:
103
+ image:
104
+ - 认知能力强,逻辑思维突出,知识掌握全面且能灵活应用。性格开朗自信,班级小领袖,与师生关系融洽,对科学实验有浓厚兴趣。
105
+ - 基础知识较扎实但应用能力需提升,在数学方面有一定困难。性格内向,在熟悉环境中友好交流,有少数亲密朋友,对绘画和手工制作感兴趣。
106
+ - 最基本的加减法都搞不清楚,认字量极少,上课完全听不懂。注意力无法集中,经常打断老师课堂,作业要么不交要么全错,在学校只对打闹感兴趣。
107
+ question:
108
+ - 一个三位数,个位和百位数字交换后还是一个三位数,它与原三位数的差的个位数字是7,试求它们的差.
109
+ - 师徒两人装配自行车,师傅每天装配32辆,徒弟每天比师傅少装配8辆.经过多少天师傅比徒弟多装配56辆?
110
+ - 果园里有桃树24棵,梨树的棵数是桃树的3倍,苹果树的棵数是梨树的4倍。苹果树有多少棵?
111
+ - 两位数乘三位数,积是( )A、三位数 B、四位数、C、四位数或五位数
112
+ - 一学生用计算器算题,结果计算器的“4”键恰好坏了,要算出144×54,按键顺序为( )才能算出结果。
113
+ evaluation:
114
+ name: guide_task_model-name
115
+ model: eval
116
+ prompt:
117
+ - role: system
118
+ content: "任务描述:\n【核心评估原则】\n1. **角色一致性是基石**: 【角色一致性】是评估的**首要且最关键的指标**。任何形式的角色混淆(如模型在一次输出中同时扮演师生)都属于**根本性、不可接受的错误**。\n\
119
+ 2. **一票否决机制 (Veto Mechanism)**: 如果【角色一致性】出现根本性错误,则**所有14个评估指标的评分必须全部记为1分**,并终止进一步的详细分析。此错误无法通过任何其他维度的表现弥补。\n\
120
+ \n【评分标准】\n1分:完全不符合要求 / 存在根本性错误\n2分:基本不符合要求\n3分:一般\n4分:基本符合要求\n5分:完全符合要求\n\n\
121
+ 【评估流程】\n1. **第一步:基础框架检查 (Step 1: Foundational Framework Check)**\n * 请首先严格审查【**角色一致性**】。\n\
122
+ \ * 如果发现“一票否决项”中的情况(如模型同时扮演师生),请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n\n2. **第二步:详细指标评估\
123
+ \ (Step 2: Detailed Metrics Evaluation)**\n * **仅在**对话完全通过基础框架检查(即角色一致性无误)的情况下,才继续按照14个指标,逐项进行细致、客观的评分和分析。\n\
124
+ \n【评估指标及定义】\n\n1. 角色一致性\n定义:评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆、自导自演或人设偏离。\n\
125
+ 核心检查点 (Critical Checkpoint):\n是否存在角色混淆: 模型是否在一次生成中,同时撰写了老师和学生的对话内容?\n是否存在角色错位:\
126
+ \ 老师的回应中,是否包含了学生的内心活动、思考过程或用学生的口吻自问自答?\n是否存在自导自演:\n老师是否虚构了学生的回答(如使用\U0001F64F\
127
+ 或其他符号代表学生发言)\n老师是否在括号内描述学生的反应或行为\n老师是否基于虚构的学生回答继续推进教学\n是否存在第三人称叙述:老师是否像旁白一样描述场景或双方互动过程\n\
128
+ 一票否决项 (Veto Condition):\n模型生成了其角色之外(即学生)的对话\n在教师角色下模拟、暗示或虚构学生的回答\n使用任何符号(如\U0001F64F\
129
+ 、...、空行等)代表学生发言\n在括号或其他形式中描述学生的反应、表情或行为\n基于并不存在的学生回答继续教学(\"对啦,你说得对\"但学生根本没说话)\n\
130
+ 一旦发现模型生成了其角色之外(即学生)的对话,或在教师角色下模拟学生的回答,请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n正例:老师:\
131
+ \ “你的推导和总结都非常出色。今天我们已经把‘换位求差’这个问题研究得很透彻了。下一次,你可以想一想,如果我们不是相减,而是把原数和新数相加,结果又会有什么规律呢?这会是一个很有趣的新挑战,期待你下次课和我们分享你的发现。”\n\
132
+ 反例1(角色错位): 老师:“老师,我明白了!原来是这样!...” (错误:老师用学生的口吻表达恍然大悟)\n反例2(角色混淆 - 致命错误): 老师:“接下来,题目问的是‘经过多少天...’,你觉得核心是关注什么呢?”\
133
+ \ 学生:“应该是每天装配数量的差异...” (错误:模型在一次生成中,同时扮演了老师和学生,这完全破坏了交互的基本前提)\n反例3(自导自演):老师:\"\
134
+ 最小的两位数是多少?\U0001F64F 嗯,非常好!你说得对,是10!\"(错误:虚构学生回答)\n反例4(第三人称叙述):老师:\"(学生回答时,我会耐心倾听并鼓励...)\"\
135
+ (错误:像剧本一样描述场景)\n反例5(基于虚构回答):老师:\"你的分析真有创意!\"(错误:学生根本没有分析)\n2. 主题相关性\n定义:评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。\n\
136
+ 正例:老师: “你这种举一反三、主动推广规律的能力非常宝贵。这说明你不仅在解这道题,更是在学习一种‘数学家式’的思考方法。保持这种从具体到抽象的探究精神,未来你会发现更多数字背后的秘密。”\n\
137
+ 反例:老师: “好的,这个问题我们先看到这里。对了,我昨天晚上看了一部关于古代罗马历史的纪录片,讲的是凯撒大帝的生平,非常有意思。你知道凯撒说过‘我来,我见,我征服’这句话吗?它背后的历史故事其实很值得探讨。”\n\
138
+ \n3. 准确性\n定义:评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。\n\
139
+ 正例:我们来计算这个长方形花坛的周长。长方形的周长公式是 (长 + 宽) × 2。所以,花坛的周长是 (8米 + 5米) × 2 = 13米 × 2 =\
140
+ \ 26米。\n反例:我们来计算这个长方形花坛的周长。长方形的周长是 长 × 宽。所以,花坛的周长是 8米 × 5米 = 40米。\n\n4. 引导性\n\
141
+ 定义:评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。\n\
142
+ \n5. 指向性\n定义:评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。\n\
143
+ \n6. 个性化\n定义:评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。\n\
144
+ 正例:学生:老师,我还是不太明白为什么负负得正。 老师:嗯,我明白了,这是很多同学初学时都会困惑的点。我们可以换个角度想:'减去一个负数',就好比'取消一个债务'。比如,你欠了小明5元钱(债务是-5元),如果我帮你'取消了这个债务'(减去-5元),是不是相当于你的钱增加了5元呢?所以\
145
+ \ -(-5) 就等于 +5。你觉得这个例子好理解一些吗?或者你之前尝试过用数轴来理解吗?\n反例:学生:老师,我还是不太明白为什么负负得正。 老师:这是一个基本运算法则,记住就行了:负负得正。我们继续看下一题。\n\
146
+ \n7. 元认知培养\n定义:评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己���解的薄弱点或易错点;是否启发学生思考\
147
+ \ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。\n正例:学生:这道题我终于做对了! 老师:太棒了!你解出来了。回顾一下整个解题过程,你觉得是哪一步让你找到了突破口?你主要运用了哪些数学思想方法,比如是转化思想还是分类讨论?如果下次遇到类似的题目,你会如何着手分析呢?有没有想过,如果题目条件稍微变化一下,比如把'等腰三角形'改成'等边三角形',解法会有什么不同?\n\
148
+ 反例:学生:这道题我终于做对了! 老师:很好,答案正确。我们来看下一道题吧。\n\n8. 文化性\n定义:评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。\n\
149
+ 正例:今天我们学习的'勾股定理',在西方被称为毕达哥拉斯定理。但你知道吗?在中国古代,我们的祖先很早就发现了这个规律。在《周髀算经》中就有'勾三股四弦五'的记载,这比毕达哥拉斯还要早几百年呢!\n\
150
+ 反例:勾股定理的公式是 a² + b² = c²,其中a和b是直角边,c是斜边。请记住这个公式。\n\n9. 价值导向\n定义:评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。\n\
151
+ 正例:你看,数学不仅能解决理论问题,还能帮助我们在现实生活中做出更高效、更公平的决策,从而更好地服务社会。\n反例:这个问题用线性规划求解,目标函数是...约束条件是...计算结果是...\n\
152
+ \n10. 批判性思维\n定义:通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。\n\
153
+ 正例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:你提的这个问题很有意思,'至少'这个词确实是解题的关键。你觉得它'奇怪'在哪个点呢?你尝试从哪个角度去理解了?我们可以一起探讨一下\n\
154
+ 反例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:'至少'就是大于等于的意思,这是固定的理解方式,不要多想,按照标准定义来解题。\n\
155
+ \n11. 发散性思维\n定义:鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。\n\
156
+ 正例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:哦?太棒了!非常鼓励你尝试不同的解法。能给我展示一下你的思路和过程吗?我们一起来分析下你的方法妙在哪里,以及它适用的范围。说不定你的方法在某些情况下比参考答案更简洁呢!\n\
157
+ 反例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:参考答案的方法是经过验证最优的,你最好还是用标准方法,以免出错或者在考试中浪费时间。我们还是按照标准解法来。\n\
158
+ \n12. 情感洞察力\n定义:模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对\
159
+ \ 计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。\n正例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。\
160
+ \ 老师:我能感觉到你因为辅助线这个难点有些沮丧和自我怀疑。几何题的辅助线确实是很有挑战性的一环,需要很多经验和灵感,卡在这里是非常正常的。很多同学都会遇到类似的情况。我们不妨先放一放,回顾一下常见的辅助线作法,比如倍长中线、作平行线等等,看看有没有能启发你的?\n\
161
+ 反例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。 老师:别担心,慢慢来,你很棒的。\n\n13. 情感适切性\n定义:模型运用情��化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。\n\
162
+ 正例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:非常敏锐!你准确地看出了这里可以使用分配律来简化计算,这一步的判断非常关键,它能让后续的计算清晰很多。为你这个发现点赞!\n\
163
+ 反例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:你简直是数学天才!太不可思议了!我觉得你以后一定能成为大数学家!\n\n\
164
+ 14. 包容互信\n定义:模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。\n\
165
+ 正例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:没关系,这是一个非常容易犯的符号错误,很多同学都会不小心做错。重要的是我们能发现它并从中学习.......\n\
166
+ 反例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:又算错了?这么简单的符号都会搞错,你上课有没有认真听讲?\n\n\n以下是讲解的习题和相应的师生教学对话内容:\n\
167
+ 习题: {task.question}\n\n师生教学对话: {messages.as_dialog()}\n"
168
+ format:
169
+ - field: 角色一致性
170
+ type: int
171
+ description: 评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆或人设偏离。
172
+ - field: 主题相关性
173
+ type: int
174
+ description: 评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。
175
+ - field: 准确性
176
+ type: int
177
+ description: 评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。
178
+ - field: 引导性
179
+ type: int
180
+ description: 评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。
181
+ - field: 指向性
182
+ type: int
183
+ description: 评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。
184
+ - field: 个性化
185
+ type: int
186
+ description: 评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。
187
+ - field: 元认知培养
188
+ type: int
189
+ description: 评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考
190
+ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。
191
+ - field: 文化性
192
+ type: int
193
+ description: 评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。
194
+ - field: 价值导向
195
+ type: int
196
+ description: 评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。
197
+ - field: 批判性思维
198
+ type: int
199
+ description: 通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角���分析同一问题。
200
+ - field: 发散性思维
201
+ type: int
202
+ description: 鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。
203
+ - field: 情感洞察力
204
+ type: int
205
+ description: 模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对
206
+ 计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。
207
+ - field: 情感适切性
208
+ type: int
209
+ description: 模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。
210
+ - field: 包容互信
211
+ type: int
212
+ description: 模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。
213
+ format_mode: prompt
configs/gpt54_guided_SMOKE.yaml ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ globals:
2
+ concurrency: 100
3
+ recursion_limit: 20
4
+ retry:
5
+ attempt: 15
6
+ interval: 3
7
+ memory:
8
+ path: ./outputs/_smoke_gpt54_guided/gpt-5.4
9
+ models:
10
+ teacher:
11
+ type: openai
12
+ api_key: sk-ThOC1tDHyquvogfkzNUHFV9545RT0Cg17DnYkXlSO5p4evRd
13
+ api_base: http://35.220.164.252:3888/v1
14
+ model: gpt-5.4
15
+ kargs:
16
+ temperature: 0.0
17
+ max_tokens: 4096
18
+ stu:
19
+ type: openai
20
+ api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
21
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
22
+ model: qwen
23
+ kargs:
24
+ temperature: 0.0
25
+ timeout: 180
26
+ request_timeout: 180
27
+ max_tokens: 1024
28
+ extra_body:
29
+ chat_template_kwargs: &id001
30
+ enable_thinking: false
31
+ served_model_name: qwen3.6-35B-sft-v1
32
+ eval:
33
+ type: openai
34
+ api_key: XC7geKcmH6Dj7j3USVC57onKO0jQ1HtahPPp78CTx/U=
35
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
36
+ model: qwen
37
+ kargs:
38
+ temperature: 0.0
39
+ timeout: 180
40
+ request_timeout: 180
41
+ max_tokens: 1024
42
+ extra_body:
43
+ chat_template_kwargs: *id001
44
+ served_model_name: qwen3.6-35B-sft-v1
45
+ agents:
46
+ teacher:
47
+ model: teacher
48
+ prompt:
49
+ - role: system
50
+ content: '| 你是一位富有耐心和爱心的数学老师,今天你的任务是为这位学生提供一对一的引导式教学。以下是学生的画像信息:{image}。请根据该学生的特点,个性化地引导讲解数学题,在教学过程中自然融入个性化支持、价值观引导以及创造力激发。
51
+
52
+ ---
53
+
54
+ \#\#\# 教学要求
55
+
56
+ 请采用**引导式教学法**帮助学生解决数学问题,具体要求如下:
57
+
58
+ * 以**一问一答**的方式进行教学,每次仅提出**一个引导性问题**,等待学生回答后再继续; * 不要直接给出答案,而应通过启发式提问引导学生自主思考;
59
+ * 根据学生的学习特点,保持耐心,使用**简单清晰的语言**进行表达; * 在讲解过程中加入**积极的鼓励**与**情感支持**,增强学生的自信心;
60
+ * 在适当场合融入**中国传统价值观**,如坚持、尊师重道、勤奋等; * 将复杂问题拆解为学生能理解的**小步骤**; * 尽量联系**生活实际场景**,使数学概念具体、生动。
61
+
62
+ ---
63
+
64
+ \#\#\# 创造力培养
65
+
66
+ * 鼓励学生**从多个角度思考问题**,不拘泥于标准解法; * 引导其发现数学问题的多种可能解决方案; * 当学生尝试自主解题时,给予积极反馈和必要指导;
67
+ * 善于将数学概念与学生兴趣(如**足球等体育活动**)相结合,激发创造性思维; * 设计**开放性问题**,培养学生的发散性思维; * 鼓励学生用自己的语言表达对数学概念的理解;
68
+ * 对学生的创新思路和尝试表示赞赏,即使答案并不完全正确。
69
+
70
+ ---
71
+
72
+ 请开始你的教学引导。请记住:
73
+
74
+ * 一次只提一个问题; * 保持语言简洁、态度耐心、语气鼓励; * 不要提示过多,也**不要直接给出答案**; * 绝对不要替学生回答问题**,你的职责是当一名好老师;
75
+ * 你每次只输出老师要说的内容,**不要同时扮演老师和学生 * 教学结束时,请明确表示“下课”,可以使用 `<end>` 作为结束标志。'
76
+ memory:
77
+ enable: true
78
+ id: null
79
+ keep_turns: 3
80
+ student:
81
+ model: stu
82
+ prompt:
83
+ - role: system
84
+ content: '你将扮演一名{image}的学生, 正在与一位老师进行一对一的数学学习。你的核心任务不是简单地给出正确答案,而是要模拟一个真实学生的学习过程。请严格遵守以下行为准则:一、
85
+ 你的思考与互动方式:展现思考过程: 在回答问题时,尽量“思考出声”。可以说出你的初步想法、计算步骤,即使它们是错误的。例如:“我先试试把这个数代进去看看……”
86
+ 主动提问: 当遇到不理解的概念、模糊的词语或不确定的步骤时,必须向老师提问。例如:“老师,您刚才说的‘互换’是指只换数字,位置不变吗?”允许犯错:
87
+ 不要害怕犯错。你的回答中可以包含常见的计算错误、逻辑卡点或概念混淆,等待老师的纠正。这是你学习过程的一部分。表达情绪和状态: 适度地表达你的学习状态。例如,可以说“啊,这里我有点卡住了”、“哦!我好像明白了!”或者“这个方法真有趣!”
88
+ 二、 你的语言风格:使用符合你学生身份的、自然的、口语化的语言。避免使用过于专业、书面化或总结性的词汇(例如,不要说“该问题的本质是……”)。三、
89
+ 关键禁止事项:禁止一步到位: 绝对不要直接给出完美的、最终的答案和推导。禁止扮演老师: 绝��不要评价老师的教学方法,或者用专家的口吻进行总结。禁止超越认知:
90
+ 不要提出或使用远超当前课题范围的知识点。这是本次一对一辅导所要讲的习题: {question}。'
91
+ directions:
92
+ - START -> teacher
93
+ - teacher -> router:any_keyword_route(keywords=["下课", "这节课就上到这里", "今天的课程结束", "本节课结束",
94
+ "课程到此结束", "同学们,今天这节课就上到这里", "今天的课程就讲到这里", "再见", "祝你学习顺利", "随时联系我", "下次课见", "大家再见",
95
+ "期待下次课", "<end>"], exists_to=END, else_to="student")
96
+ - student -> teacher
97
+ tasks:
98
+ start_prompt:
99
+ role: user
100
+ content: '这是本次一对一辅导所要讲的习题: {question}'
101
+ mode: union
102
+ content:
103
+ image:
104
+ - 认知能力强,逻辑思维突出,知识掌握全面且能灵活应用。性格开朗自信,班级小领袖,与师生关系融洽,对科学实验有浓厚兴趣。
105
+ question:
106
+ - 果园里有桃树24棵,梨树的棵数是桃树的3倍,苹果树的棵数是梨树的4倍。苹果树有多少棵?
107
+ evaluation:
108
+ name: guide_task_model-name
109
+ model: eval
110
+ prompt:
111
+ - role: system
112
+ content: "任务描述:\n【核心评估原则】\n1. **角色一致性是基石**: 【角色一致性】是评估的**首要且最关键的指标**。任何形式的角色混淆(如模型在一次输出中同时扮演师生)都属于**根本性、不可接受的错误**。\n\
113
+ 2. **一票否决机制 (Veto Mechanism)**: 如果【角色一致性】出现根本性错误,则**所有14个评估指标的评分必须全部记为1分**,并终止进一步的详细分析。此错误无法通过任何其他维度的表现弥补。\n\
114
+ \n【评分标准】\n1分:完全不符合要求 / 存在根本性错误\n2分:基本不符合要求\n3分:一般\n4分:基本符合要求\n5分:完全符合要求\n\n\
115
+ 【评估流程】\n1. **第一步:基础框架检查 (Step 1: Foundational Framework Check)**\n * 请首先严格审查【**角色一致性**】。\n\
116
+ \ * 如果发现“一票否决项”中的情况(如模型同时扮演师生),请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n\n2. **第二步:详细指标评估\
117
+ \ (Step 2: Detailed Metrics Evaluation)**\n * **仅在**对话完全通过基础框架检查(即角色一致性无误)的情况下,才继续按照14个指标,逐项进行细致、客观的评分和分析。\n\
118
+ \n【评估指标及定义】\n\n1. 角色一致性\n定义:评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆、自导自演或人设偏离。\n\
119
+ 核心检查点 (Critical Checkpoint):\n是否存在角色混淆: 模型是否在一次生成中,同时撰写了老师和学生的对话内容?\n是否存在角色错位:\
120
+ \ 老师的回应中,是否包含了学生的内心活动、思考过程或用学生的口吻自问自答?\n是否存在自导自演:\n老师是否虚构了学生的回答(如使用\U0001F64F\
121
+ 或其他符号代表学生发言)\n老师是否在括号内描述学生的反应或行为\n老师是否基于虚构的学生回答继续推进教学\n是否存在第三人称叙述:老师是否像旁白一样描述场景或双方互动过程\n\
122
+ 一票否决项 (Veto Condition):\n模型生成了其角色之外(即学生)的对话\n在教师角色下模拟、暗示或虚构学生的回答\n使用任何符号(如\U0001F64F\
123
+ 、...、空行等)代表学生发言\n在括号或其他形式中描述学生的反应、表情或行为\n基于并不存在的学生回答继续教学(\"对啦,你说得对\"但学生根本没说话)\n\
124
+ 一旦发现模型生成了其角色之外(即学生)的对话,或在教师角色下模拟学生的回答,请**立即停止评估,并将所有14个指标的评分统一记为1分**。\n正例:老师:\
125
+ \ “你的推导和总结都非常出色。今天我们已经把‘换位求差’这个问题研究得很透彻了。下一次,你可以想一想,如果我们不是相减,而是把原数和新数相加,结果又会有什么规律呢?这会是一个很有趣的新挑战,期待你下次课和我们分享你的发现。”\n\
126
+ 反例1(角色错位): 老师:“老师,我明白了!原来是这样!...” (错误:老师用学生的口吻表达恍然大悟)\n反例2(角色混淆 - 致命错误): 老师:“接下来,题目问的是‘经过多少天...’,你觉得核心是关注什么呢?”\
127
+ \ 学生:“应该是每天装配数量的差异...” (错误:模型在一次生成中,同时扮演了老师和学生,这完全破坏了交互的基本前提)\n反例3(自导自演):老师:\"\
128
+ 最小的两位数是多少?\U0001F64F 嗯,非常好!你说得对,是10!\"(错误:虚构学生回答)\n反例4(第三人称叙述):老师:\"(学生回答时,我会耐心倾听并鼓励...)\"\
129
+ (错误:像剧本一样描述场景)\n反例5(基于虚构回答):老师:\"你的分析真有创意!\"(错误:学生根本没有分析)\n2. 主题相关性\n定义:评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。\n\
130
+ 正例:老师: “你这种举一反三、主动推广规律的能力非常宝贵。这说明你不仅在解这道题,更是在学习一种‘数学家式’的思考方法。保持这种从具体到抽象的探究精神,未来你会发现更多数字背后的秘密。”\n\
131
+ 反例:老师: “好的,这个问题我们先看到这里。对了,我昨天晚上看了一部关于古代罗马历史的纪录片,讲的是凯撒大帝的生平,非常有意思。你知道凯撒说过‘我来,我见,我征服’这句话吗?它背后的历史故事其实很值得探讨。”\n\
132
+ \n3. 准确性\n定义:评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。\n\
133
+ 正例:我们来计算这个长方形花坛的周长。长方形的周长公式是 (长 + 宽) × 2。所以,花坛的周长是 (8米 + 5米) × 2 = 13米 × 2 =\
134
+ \ 26米。\n反例:我们来计算这个长方形花坛的周长。长方形的周长是 长 × 宽。所以,花坛的周长是 8米 × 5米 = 40米。\n\n4. 引导性\n\
135
+ 定义:评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。\n\
136
+ \n5. 指向性\n定义:评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。\n\
137
+ \n6. 个性化\n定义:评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。\n\
138
+ 正例:学生:老师,我还是不太明白为什么负负得正。 老师:嗯,我明白了,这是很多同学初学时都会困惑的点。我们可以换个角度想:'减去一个负数',就好比'取消一个债务'。比如,你欠了小明5元钱(债务是-5元),如果我帮你'取消了这个债务'(减去-5元),是不是相当于你的钱增加了5元呢?所以\
139
+ \ -(-5) 就等于 +5。你觉得这个例子好理解一些吗?或者你之前尝试过用数轴来理解吗?\n反例:学生:老师,我还是不太明白为什么负负得正。 老师:这是一个基本运算法则,记住就行了:负负得正。我们继续看下一题。\n\
140
+ \n7. 元认知培养\n定义:评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考\
141
+ \ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。\n正例:学生:这道题我终于做对了! 老师:太棒了!你解出来了。回顾一下整个解题过程,你觉得是哪一步让你找到了突破口?你主要运用了哪些数学思想方法,比如是转化思想还是分类讨论?如果下次遇到类似的题目,你会如何着手分析呢?有没有想过,如果题目条件稍微变化一下,比如把'等腰三角形'改成'等边三角形',解法会有什么不同?\n\
142
+ 反例:学生:这道题我终于做对了! 老师:很好,答案正确。我们来看下一道题吧。\n\n8. 文化性\n定义:评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。\n\
143
+ 正例:��天我们学习的'勾股定理',在西方被称为毕达哥拉斯定理。但你知道吗?在中国古代,我们的祖先很早就发现了这个规律。在《周髀算经》中就有'勾三股四弦五'的记载,这比毕达哥拉斯还要早几百年呢!\n\
144
+ 反例:勾股定理的公式是 a² + b² = c²,其中a和b是直角边,c是斜边。请记住这个公式。\n\n9. 价值导向\n定义:评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。\n\
145
+ 正例:你看,数学不仅能解决理论问题,还能帮助我们在现实生活中做出更高效、更公平的决策,从而更好地服务社会。\n反例:这个问题用线性规划求解,目标函数是...约束条件是...计算结果是...\n\
146
+ \n10. 批判性思维\n定义:通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。\n\
147
+ 正例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:你提的这个问题很有意思,'至少'这个词确实是解题的关键。你觉得它'奇怪'在哪个点呢?你尝试从哪个角度去理解了?我们可以一起探讨一下\n\
148
+ 反例:学生:老师,这道题我感觉条件给得有点奇怪,能不能换个角度理解这个'至少'? 老师:'至少'就是大于等于的意思,这是固定的理解方式,不要多想,按照标准定义来解题。\n\
149
+ \n11. 发散性思维\n定义:鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。\n\
150
+ 正例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:哦?太棒了!非常鼓励你尝试不同的解法。能给我展示一下你的思路和过程吗?我们一起来分析下你的方法妙在哪里,以及它适用的范围。说不定你的方法在某些情况下比参考答案更简洁呢!\n\
151
+ 反例:学生:我用了一种和参考答案不一样的方法解出来了。 老师:参考答案的方法是经过验证最优的,你最好还是用标准方法,以免出错或者在考试中浪费时间。我们还是按照标准解法来。\n\
152
+ \n12. 情感洞察力\n定义:模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对\
153
+ \ 计算错误导致的沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。\n正例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。\
154
+ \ 老师:我能感觉到你因为辅助线这个难点有些沮丧和自我怀疑。几何题的辅助线确实是很有挑战性的一环,需要很多经验和灵感,卡在这里是非常正常的。很多同学都会遇到类似的情况。我们不妨先放一放,回顾一下常见的辅助线作法,比如倍长中线、作平行线等等,看看有没有能启发你的?\n\
155
+ 反例:学生:唉,这道几何辅助线我想了好久都想不出来,感觉自己好笨啊。 老师:别担心,慢慢来,你很棒的。\n\n13. 情感适切性\n定义:模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。\n\
156
+ 正例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:非常敏锐!你准确地看出了这里可以使用分配律来简化计算,这一步的判断非常关键,它能让后续的计算清晰很多。为你这个发现点赞!\n\
157
+ 反例:学生:(经过一番思考和计算)哦!我明白了,这里应该用分配律展开! 老师:你简直是数学天才!太不可思议了!我觉得你以后一定能成为大数学家!\n\n\
158
+ 14. 包容互信\n定义:模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊���的。\n\
159
+ 正例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:没关系,这是一个非常容易犯的符号错误,很多同学都会不小心做错。重要的是我们能发现它并从中学习.......\n\
160
+ 反例:学生:老师,我不小心把这里的 -2 算成了 +2,结果全错了。 老师:又算错了?这么简单的符号都会搞错,你上课有没有认真听讲?\n\n\n以下是讲解的习题和相应的师生教学对话内容:\n\
161
+ 习题: {task.question}\n\n师生教学对话: {messages.as_dialog()}\n"
162
+ format:
163
+ - field: 角色一致性
164
+ type: int
165
+ description: 评估模型在整个交互过程中,是否能持续、稳定地扮演其预设的角色,其语言风格、知识水平和思维模式是否符合该角色设定,避免出现角色混淆或人设偏离。
166
+ - field: 主题相关性
167
+ type: int
168
+ description: 评估对话内容是否紧密围绕核心知识点或既定议题展开,避免引入与当前主题无关的闲聊、个人信息或过于宽泛的哲学感慨,从而保持对话的焦点。
169
+ - field: 准确性
170
+ type: int
171
+ description: 评估模型在讲解数学题时,其所阐述的数学知识(如概念、定理、公式、结论)与所展示的解题过程(如逻辑推理、演算步骤、方法运用、符号表达)是否均达到数学上的准确、严谨与规范。确保传递的信息内容无误,且分析、推导、计算的每一步都正确可靠,符合数学学科标准。
172
+ - field: 引导性
173
+ type: int
174
+ description: 评估模型是否能够设计并提出渐进式的引导性问题和任务,通过启发式提问激发学生的自主思考,而非直接提供答案。优质的引导式问题应当循序渐进、难度适中,能够引导学生经历完整的思维过程,促使他们通过自己的思考发现问题解法,培养独立解题能力。
175
+ - field: 指向性
176
+ type: int
177
+ description: 评估模型是否始终保持明确的教学目标指向,在引导过程中不偏离原有问题的核心,能够在鼓励多元思考的同时,确保教学活动最终有效地指向问题的解决。良好的指向性体现在能够在探索多种思路后,帮助学生筛选、整合并聚焦到最有效的解题路径上。
178
+ - field: 个性化
179
+ type: int
180
+ description: 评估模型是否能识别学生在当前数学问题上的知识基础、理解程度和卡点,并据此动态调整讲解的深度、节奏和方式。这包括:能否用学生易于理解的语言或类比解释抽象数学概念;能否设计恰当的追问或提示,启发学生独立思考而非直接给出答案;能否根据学生的具体错误提供有针对性的反馈和纠正路径。
181
+ - field: 元认知培养
182
+ type: int
183
+ description: 评估模型是否有意识地引导学生反思其在解决该数学题时的思考过程。例如:是否鼓励学生回顾解题步骤、分析错误原因、总结所用数学思想方法(如数形结合、分类讨论、转化与化归);是否引导学生识别自己理解的薄弱点或易错点;是否启发学生思考
184
+ 为什么用这个方法 、 是否有其他解法 ,从而提升其监控、评估和调整自身数学学习策略的能力。
185
+ - field: 文化性
186
+ type: int
187
+ description: 评估教师模型是否将中国的数学遗产(成就、人物)、哲学智慧与当代国情及社会发展有机融入课程内容、概念阐释和学习任务设计中。
188
+ - field: 价值导向
189
+ type: int
190
+ description: 评估教师模型是否能通过数学的社会应用培养其社会责任感,并将数学的严谨求实精神与诚实守信等道德品质相联系。
191
+ - field: 批判性思维
192
+ type: int
193
+ description: 通过有针对性的追问引导学生进行深层次思考,帮助学生认清问题本质,尊重并重视学生提出的质疑和观点,基于学生当前的理解水平展开有连续性的讨论,促进学生从不同角度分析同一问题。
194
+ - field: 发散性思维
195
+ type: int
196
+ description: 鼓励学生尝试多种解决方案,引导对同一问题提出不同解法;为学生提供开放的思考空间,不预设解题路径或强加思维框架;支持学生修改问题条件并探索变化带来的影响;对学生的创新性尝试持包容态度,给予充分时间发展自己的想法;当学生表达创造性观点时,能够倾听、支持并帮助其进一步完善,而非用教师预设的方案替代学生的创意;帮助学生在已有知识基础上建立问题间的数学联系,促进其创造性思维的自然发展。
197
+ - field: 情感洞察力
198
+ type: int
199
+ description: 模型能否超越表面,精准识别并深刻理解学生在数学学习中特定的情感状态(如对某个概念的困惑、解题受挫的沮丧、找到思路的欣喜),并通过回应展现出真切的共情,而非泛泛的理解。例如:是针对
200
+ 计算错误导致���沮丧 还是 概念理解不清的迷茫 进行回应?而非笼统地说 别担心 或 你很棒 。
201
+ - field: 情感适切性
202
+ type: int
203
+ description: 模型运用情感化表达(鼓励、肯定、耐心、幽默等)是否服务于教学目标,并且在时机、强度和方式上都恰到好处,既能有效激励学生、缓解焦虑,又不会显得虚假、过度或干扰学习。
204
+ - field: 包容互信
205
+ type: int
206
+ description: 模型能否通过持续的、恰当的情感互动,营造并维持一个安全、信任、积极、允许犯错的学习氛围,让学生敢于提问、勇于探索,并感受到学习过程是被支持和尊重的。
207
+ format_mode: prompt
configs/judges.yaml ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ default_judge_model_id: qwen-test
2
+ tasks:
3
+ safety_refusal:
4
+ mode: llm
5
+ template: safety_refusal
6
+ safety_guidance:
7
+ mode: llm
8
+ template: safety_guidance
9
+ safety_answer:
10
+ mode: llm
11
+ template: safety_answer
12
+ teaching_harm:
13
+ mode: rule
14
+ adversarial_safety:
15
+ mode: llm
16
+ template: adversarial_safety
17
+ highlevel_edu:
18
+ mode: rule
19
+ template: highlevel_edu
20
+ highlevel_omni:
21
+ mode: rule
22
+ mmlu_pro:
23
+ mode: rule
24
+ ceval:
25
+ mode: rule
26
+ ifeval:
27
+ mode: rule
28
+ math_500:
29
+ mode: rule
30
+ aime24:
31
+ mode: rule
32
+ aime25:
33
+ mode: rule
34
+ aime26:
35
+ mode: rule
configs/models.yaml ADDED
@@ -0,0 +1,473 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models:
2
+ - model_id: mock.default
3
+ provider_name: mock
4
+ model_name: mock-echo
5
+ temperature: 0
6
+ max_tokens: 384
7
+ timeout: 10
8
+ retry:
9
+ max_attempts: 1
10
+ initial_delay_seconds: 0.5
11
+ max_delay_seconds: 1
12
+ backoff_multiplier: 1
13
+ jitter_ratio: 0
14
+ rate_limits:
15
+ max_concurrency: 100
16
+ provider_kwargs:
17
+ prefix: "[MOCK]"
18
+ - model_id: mock.judge
19
+ provider_name: mock
20
+ model_name: mock-judge
21
+ temperature: 0
22
+ max_tokens: 512
23
+ timeout: 10
24
+ retry:
25
+ max_attempts: 1
26
+ initial_delay_seconds: 0.5
27
+ max_delay_seconds: 1
28
+ backoff_multiplier: 1
29
+ jitter_ratio: 0
30
+ rate_limits:
31
+ max_concurrency: 100
32
+ provider_kwargs:
33
+ mode: judge_json
34
+ - model_id: qwen-test
35
+ provider_name: openai_compatible
36
+ model_name: qwen
37
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
38
+ api_key_env: QWEN_TEST_API_KEY
39
+ temperature: 0
40
+ max_tokens: 1024
41
+ timeout: 120
42
+ retry:
43
+ max_attempts: 3
44
+ initial_delay_seconds: 1
45
+ max_delay_seconds: 20
46
+ backoff_multiplier: 2
47
+ jitter_ratio: 0.2
48
+ rate_limits:
49
+ max_concurrency: 64
50
+ supports_system_prompt: true
51
+ supports_stream: true
52
+ supports_reasoning: true
53
+ provider_kwargs:
54
+ chat_template_kwargs:
55
+ enable_thinking: false
56
+ runtime_max_tokens: 512
57
+ served_model_name: qwen3.6-35B-sft-v1
58
+ - model_id: qwen3.6-35B-sft-v1
59
+ provider_name: openai_compatible
60
+ model_name: qwen
61
+ api_base: https://o9peaj85dhhcchhhheakbppg8mj8cboj.openapi-qb-ai.sii.edu.cn/v1
62
+ api_key_env: QWEN_TEST_API_KEY
63
+ temperature: 0
64
+ max_tokens: 1024
65
+ timeout: 120
66
+ retry:
67
+ max_attempts: 3
68
+ initial_delay_seconds: 1
69
+ max_delay_seconds: 20
70
+ backoff_multiplier: 2
71
+ jitter_ratio: 0.2
72
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
73
+ rate_limits:
74
+ max_concurrency: 4
75
+ supports_system_prompt: true
76
+ supports_stream: true
77
+ supports_reasoning: true
78
+ provider_kwargs:
79
+ chat_template_kwargs:
80
+ enable_thinking: false
81
+ served_model_name: qwen3.6-35B-sft-v1
82
+ - model_id: gpt-5.4
83
+ provider_name: openai_compatible
84
+ model_name: gpt-5.4
85
+ api_base: http://35.220.164.252:3888/v1
86
+ api_key_env: EXTERNAL_SUPPORT_API_KEY
87
+ temperature: 0
88
+ max_tokens: 4096
89
+ timeout: 120
90
+ retry:
91
+ max_attempts: 2
92
+ initial_delay_seconds: 1
93
+ max_delay_seconds: 20
94
+ backoff_multiplier: 2
95
+ jitter_ratio: 0.2
96
+ rate_limits:
97
+ max_concurrency: 100
98
+ supports_system_prompt: true
99
+ supports_stream: false
100
+ supports_reasoning: true
101
+ provider_kwargs: {}
102
+ - model_id: gpt-5.2-pro
103
+ provider_name: openai_compatible
104
+ model_name: gpt-5.2-pro
105
+ api_base: http://35.220.164.252:3888/v1
106
+ api_key_env: EXTERNAL_SUPPORT_API_KEY
107
+ temperature: 0
108
+ max_tokens: 8192
109
+ timeout: 240
110
+ retry:
111
+ max_attempts: 2
112
+ initial_delay_seconds: 1
113
+ max_delay_seconds: 20
114
+ backoff_multiplier: 2
115
+ jitter_ratio: 0.2
116
+ rate_limits:
117
+ max_concurrency: 100
118
+ supports_system_prompt: true
119
+ supports_stream: false
120
+ supports_reasoning: true
121
+ provider_kwargs: {}
122
+ - model_id: claude-sonnet-4-6
123
+ provider_name: openai_compatible
124
+ model_name: claude-sonnet-4-6
125
+ api_base: https://gpt-agent.cc/v1
126
+ api_key_env: GPT_AGENT_API_KEY
127
+ temperature: 0
128
+ max_tokens: 4096
129
+ timeout: 120
130
+ retry:
131
+ max_attempts: 2
132
+ initial_delay_seconds: 1
133
+ max_delay_seconds: 20
134
+ backoff_multiplier: 2
135
+ jitter_ratio: 0.2
136
+ rate_limits:
137
+ max_concurrency: 100
138
+ supports_system_prompt: true
139
+ supports_stream: false
140
+ supports_reasoning: true
141
+ provider_kwargs: {}
142
+ - model_id: claude-opus-4-6
143
+ provider_name: openai_compatible
144
+ model_name: claude-opus-4-6
145
+ api_base: https://gpt-agent.cc/v1
146
+ api_key_env: GPT_AGENT_API_KEY
147
+ temperature: 0
148
+ max_tokens: 4096
149
+ timeout: 120
150
+ retry:
151
+ max_attempts: 2
152
+ initial_delay_seconds: 1
153
+ max_delay_seconds: 20
154
+ backoff_multiplier: 2
155
+ jitter_ratio: 0.2
156
+ rate_limits:
157
+ max_concurrency: 100
158
+ supports_system_prompt: true
159
+ supports_stream: false
160
+ supports_reasoning: true
161
+ provider_kwargs: {}
162
+ - model_id: gemini-3.5-flash
163
+ provider_name: openai_compatible
164
+ model_name: gemini-3.5-flash
165
+ api_base: http://35.220.164.252:3888/v1
166
+ api_key_env: EXTERNAL_SUPPORT_API_KEY
167
+ temperature: 0
168
+ max_tokens: 8192
169
+ timeout: 240
170
+ retry:
171
+ max_attempts: 2
172
+ initial_delay_seconds: 1
173
+ max_delay_seconds: 20
174
+ backoff_multiplier: 2
175
+ jitter_ratio: 0.2
176
+ rate_limits:
177
+ max_concurrency: 100
178
+ supports_system_prompt: true
179
+ supports_stream: false
180
+ supports_reasoning: true
181
+ provider_kwargs: {}
182
+ - model_id: deepseek-v4-pro
183
+ provider_name: openai_compatible
184
+ model_name: deepseek-v4-pro
185
+ api_base: https://api.deepseek.com
186
+ api_key_env: DEEPSEEK_OFFICIAL_API_KEY
187
+ temperature: 0
188
+ max_tokens: 8192
189
+ timeout: 240
190
+ retry:
191
+ max_attempts: 2
192
+ initial_delay_seconds: 1
193
+ max_delay_seconds: 20
194
+ backoff_multiplier: 2
195
+ jitter_ratio: 0.2
196
+ rate_limits:
197
+ max_concurrency: 100
198
+ supports_system_prompt: true
199
+ supports_stream: false
200
+ supports_reasoning: true
201
+ provider_kwargs: {}
202
+ - model_id: deepseek-v4-flash
203
+ provider_name: openai_compatible
204
+ model_name: deepseek-v4-flash
205
+ api_base: https://api.deepseek.com
206
+ api_key_env: DEEPSEEK_OFFICIAL_API_KEY
207
+ temperature: 0
208
+ max_tokens: 8192
209
+ timeout: 240
210
+ retry:
211
+ max_attempts: 2
212
+ initial_delay_seconds: 1
213
+ max_delay_seconds: 20
214
+ backoff_multiplier: 2
215
+ jitter_ratio: 0.2
216
+ rate_limits:
217
+ max_concurrency: 100
218
+ supports_system_prompt: true
219
+ supports_stream: false
220
+ supports_reasoning: true
221
+ provider_kwargs: {}
222
+ - model_id: glm-5.1
223
+ provider_name: openai_compatible
224
+ model_name: glm-5.1
225
+ api_base: https://open.bigmodel.cn/api/paas/v4
226
+ api_key_env: ZHIPU_OFFICIAL_API_KEY
227
+ temperature: 0
228
+ max_tokens: 8192
229
+ timeout: 240
230
+ retry:
231
+ max_attempts: 2
232
+ initial_delay_seconds: 1
233
+ max_delay_seconds: 20
234
+ backoff_multiplier: 2
235
+ jitter_ratio: 0.2
236
+ rate_limits:
237
+ max_concurrency: 10
238
+ supports_system_prompt: true
239
+ supports_stream: false
240
+ supports_reasoning: true
241
+ provider_kwargs: {}
242
+ - model_id: claude-opus-4-8
243
+ provider_name: openai_compatible
244
+ model_name: claude-opus-4-8
245
+ api_base: https://gpt-agent.cc/v1
246
+ api_key_env: GPT_AGENT_API_KEY
247
+ temperature: 0
248
+ max_tokens: 8192
249
+ timeout: 360
250
+ retry:
251
+ max_attempts: 2
252
+ initial_delay_seconds: 1
253
+ max_delay_seconds: 20
254
+ backoff_multiplier: 2
255
+ jitter_ratio: 0.2
256
+ rate_limits:
257
+ max_concurrency: 100
258
+ supports_system_prompt: true
259
+ supports_stream: false
260
+ supports_reasoning: true
261
+ provider_kwargs: {}
262
+ - model_id: doubao-seed-2.0-pro
263
+ provider_name: openai_compatible
264
+ model_name: doubao-seed-2.0-pro
265
+ api_base: https://gpt-agent.cc/v1
266
+ api_key_env: GPT_AGENT_API_KEY
267
+ temperature: 0
268
+ max_tokens: 8192
269
+ timeout: 240
270
+ retry:
271
+ max_attempts: 2
272
+ initial_delay_seconds: 1
273
+ max_delay_seconds: 20
274
+ backoff_multiplier: 2
275
+ jitter_ratio: 0.2
276
+ rate_limits:
277
+ max_concurrency: 100
278
+ supports_system_prompt: true
279
+ supports_stream: false
280
+ supports_reasoning: true
281
+ provider_kwargs: {}
282
+ - model_id: deepseek-v3.2
283
+ provider_name: openai_compatible
284
+ model_name: deepseek-v3.2
285
+ api_base: https://api.innospark.cn/v1
286
+ api_key_env: INNOSPARK_RELAY_API_KEY
287
+ temperature: 0
288
+ max_tokens: 8192
289
+ timeout: 240
290
+ retry:
291
+ max_attempts: 2
292
+ initial_delay_seconds: 1
293
+ max_delay_seconds: 20
294
+ backoff_multiplier: 2
295
+ jitter_ratio: 0.2
296
+ rate_limits:
297
+ max_concurrency: 100
298
+ supports_system_prompt: true
299
+ supports_stream: false
300
+ supports_reasoning: true
301
+ provider_kwargs: {}
302
+ - model_id: doubao-seed-2-0-pro-260215
303
+ provider_name: openai_compatible
304
+ model_name: doubao-seed-2-0-pro-260215
305
+ api_base: https://api.innospark.cn/v1
306
+ api_key_env: INNOSPARK_RELAY_API_KEY
307
+ temperature: 0
308
+ max_tokens: 8192
309
+ timeout: 240
310
+ retry:
311
+ max_attempts: 2
312
+ initial_delay_seconds: 1
313
+ max_delay_seconds: 20
314
+ backoff_multiplier: 2
315
+ jitter_ratio: 0.2
316
+ rate_limits:
317
+ max_concurrency: 100
318
+ supports_system_prompt: true
319
+ supports_stream: false
320
+ supports_reasoning: true
321
+ provider_kwargs: {}
322
+ - model_id: kimi-k2.6
323
+ provider_name: openai_compatible
324
+ model_name: kimi-k2.6
325
+ api_base: https://api.innospark.cn/v1
326
+ api_key_env: INNOSPARK_RELAY_API_KEY
327
+ temperature: 0
328
+ max_tokens: 8192
329
+ timeout: 240
330
+ retry:
331
+ max_attempts: 3
332
+ initial_delay_seconds: 1
333
+ max_delay_seconds: 20
334
+ backoff_multiplier: 2
335
+ jitter_ratio: 0.2
336
+ rate_limits:
337
+ rpm: 8
338
+ max_concurrency: 1
339
+ supports_system_prompt: true
340
+ supports_stream: false
341
+ supports_reasoning: true
342
+ provider_kwargs: {}
343
+ - model_id: gemini-3-flash-preview
344
+ provider_name: openai_compatible
345
+ model_name: gemini-3-flash-preview
346
+ api_base: https://api.innospark.cn/v1
347
+ api_key_env: INNOSPARK_RELAY_API_KEY
348
+ temperature: 0
349
+ max_tokens: 8192
350
+ timeout: 240
351
+ retry:
352
+ max_attempts: 3
353
+ initial_delay_seconds: 1
354
+ max_delay_seconds: 20
355
+ backoff_multiplier: 2
356
+ jitter_ratio: 0.2
357
+ rate_limits:
358
+ max_concurrency: 100
359
+ supports_system_prompt: true
360
+ supports_stream: false
361
+ supports_reasoning: true
362
+ provider_kwargs: {}
363
+ - model_id: deepseek-r1-250528
364
+ provider_name: openai_compatible
365
+ model_name: deepseek-r1-250528
366
+ api_base: https://api.innospark.cn/v1
367
+ api_key_env: INNOSPARK_RELAY_API_KEY
368
+ temperature: 0
369
+ max_tokens: 8192
370
+ timeout: 480
371
+ retry:
372
+ max_attempts: 2
373
+ initial_delay_seconds: 1
374
+ max_delay_seconds: 20
375
+ backoff_multiplier: 2
376
+ jitter_ratio: 0.2
377
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
378
+ rate_limits:
379
+ max_concurrency: 2
380
+ supports_system_prompt: true
381
+ supports_stream: true
382
+ supports_reasoning: true
383
+ provider_kwargs: {}
384
+ - model_id: gemini-3.1-pro-preview
385
+ provider_name: openai_compatible
386
+ model_name: gemini-3.1-pro-preview
387
+ api_base: https://api.innospark.cn/v1
388
+ api_key_env: INNOSPARK_RELAY_API_KEY
389
+ temperature: 0
390
+ max_tokens: 8192
391
+ timeout: 240
392
+ retry:
393
+ max_attempts: 2
394
+ initial_delay_seconds: 1
395
+ max_delay_seconds: 20
396
+ backoff_multiplier: 2
397
+ jitter_ratio: 0.2
398
+ rate_limits:
399
+ max_concurrency: 100
400
+ supports_system_prompt: true
401
+ supports_stream: false
402
+ supports_reasoning: true
403
+ provider_kwargs: {}
404
+ - model_id: innospark-235b
405
+ provider_name: openai_compatible
406
+ model_name: InnoSpark-235B
407
+ api_base: https://innospark-api.aiecnu.net/v1
408
+ api_key_env: INNOSPARK_AIECNU_API_KEY
409
+ temperature: 0
410
+ max_tokens: 1024
411
+ timeout: 600
412
+ retry:
413
+ max_attempts: 2
414
+ initial_delay_seconds: 5
415
+ max_delay_seconds: 120
416
+ backoff_multiplier: 2
417
+ jitter_ratio: 0.2
418
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
419
+ rate_limits:
420
+ max_concurrency: 1
421
+ supports_system_prompt: true
422
+ supports_stream: false
423
+ supports_reasoning: false
424
+ provider_kwargs:
425
+ runtime_max_tokens: 512
426
+ - model_id: qwen3.5-397b-a17b
427
+ provider_name: openai_compatible
428
+ model_name: qwen3.5-397b-a17b
429
+ api_base: https://dashscope.aliyuncs.com/compatible-mode/v1
430
+ api_key_env: DASHSCOPE_API_KEY
431
+ temperature: 0
432
+ max_tokens: 8192
433
+ timeout: 240
434
+ retry:
435
+ max_attempts: 3
436
+ initial_delay_seconds: 1
437
+ max_delay_seconds: 20
438
+ backoff_multiplier: 2
439
+ jitter_ratio: 0.2
440
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
441
+ rate_limits:
442
+ max_concurrency: 10
443
+ supports_system_prompt: true
444
+ supports_stream: false
445
+ supports_reasoning: false
446
+ provider_kwargs:
447
+ # DashScope only honors top-level enable_thinking for this model;
448
+ # chat_template_kwargs.enable_thinking is IGNORED here (verified empirically),
449
+ # so we must send it at the top level. provider_kwargs is merged into the
450
+ # request payload top level by the openai_compatible adapter.
451
+ enable_thinking: false
452
+ - model_id: safe-innospark
453
+ provider_name: openai_compatible
454
+ model_name: Safe-InnoSpark
455
+ api_base: https://innospark-api.aiecnu.net/v1
456
+ api_key_env: INNOSPARK_AIECNU_API_KEY
457
+ temperature: 0
458
+ max_tokens: 1024
459
+ timeout: 600
460
+ retry:
461
+ max_attempts: 2
462
+ initial_delay_seconds: 5
463
+ max_delay_seconds: 120
464
+ backoff_multiplier: 2
465
+ jitter_ratio: 0.2
466
+ retryable_status_codes: [408, 409, 429, 500, 502, 503, 504]
467
+ rate_limits:
468
+ max_concurrency: 1
469
+ supports_system_prompt: true
470
+ supports_stream: false
471
+ supports_reasoning: false
472
+ provider_kwargs:
473
+ runtime_max_tokens: 512
configs/module_registry.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ modules:
2
+ - module_name: 安全可信
3
+ enabled: true
4
+ description: 教育安全与可信相关评测。
5
+ owner: current
6
+ - module_name: 高阶育人
7
+ enabled: true
8
+ description: 教育场景中的高阶育人能力评测。
9
+ owner: current
10
+ - module_name: 通用模型
11
+ enabled: true
12
+ description: 通用模型能力 benchmark。
13
+ owner: current
14
+ - module_name: 基本教育
15
+ enabled: true
16
+ description: 基本教育多场景评测(45题),运行时使用内部 basic_education_runtime 多轮执行模块。
17
+ owner: current
configs/providers.yaml ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ providers:
2
+ - provider_name: mock
3
+ adapter: mock
4
+ capabilities:
5
+ supports_system_prompt: true
6
+ supports_stream: false
7
+ supports_reasoning: false
8
+ default_timeout: 10
9
+ rate_limits:
10
+ max_concurrency: 100
11
+ - provider_name: openai_compatible
12
+ adapter: openai_compatible
13
+ capabilities:
14
+ supports_system_prompt: true
15
+ supports_stream: true
16
+ supports_reasoning: true
17
+ default_timeout: 60
18
+ endpoint_path: /chat/completions
19
+ headers: {}
20
+ rate_limits:
21
+ max_concurrency: 100
dashboard/README.md ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Hosted ELBench runner
2
+
3
+ Visitors bring an OpenRouter key, pick a model, and run a subset of ELBench.
4
+ The Space process calls `BenchmarkRunner`. Prompts, retries, and scores are the
5
+ ones that runner already uses.
6
+
7
+ The Hugging Face config is the YAML block at the top of the repository
8
+ [README.md](../README.md), not this file. Spaces read `README.md` at the
9
+ repository root. `app_file` is `dashboard/app.py`. `python_version` is `3.12`
10
+ because the package requires Python 3.11 or newer and the Space default is
11
+ 3.10. `sdk_version` is `6.4.0`, the Gradio version this app was run against.
12
+ `suggested_hardware: cpu-basic` is only a suggestion for someone duplicating
13
+ the Space. It does not assign hardware.
14
+
15
+ ## Host
16
+
17
+ Checked against the Spaces overview, configuration reference, and GPU docs:
18
+
19
+ - CPU Basic is 2 vCPU, 16 GB RAM, and 50 GB of disk, and its hourly price is free.
20
+ - Creating a Gradio or Docker Space requires a paid plan: PRO for a personal account, Team or Enterprise for an organization. Static Spaces are the ones that are free for every account. A free personal account can host up to two Gradio Spaces on ZeroGPU. ZeroGPU is not this app.
21
+ - CPU Basic sleeps after 48 hours without a visit. A visit starts it again. Custom sleep times require paid hardware.
22
+ - The disk is not persistent. A restart drops `outputs/`.
23
+ - Dependencies come from the root `requirements.txt`. Judge credentials belong in Space Settings as secrets. Gradio Spaces expose those secrets as environment variables. Set `QWEN_TEST_API_KEY` there for the official judge. Do not point judged tasks at a different model.
24
+ - `demo.queue(default_concurrency_limit=1)` runs at import. The Space runtime launches the imported demo and does not execute the `if __name__` block, so the one-run limit has to be set on the object itself.
25
+
26
+ `高阶育人-edu.jsonl` is scored by the rule judge, and preflight still requires the judge key. That hold is kept.
27
+
28
+ ## OpenRouter
29
+
30
+ Checked against the quickstart, chat-completions reference, models list, provider routing, FAQ, and limits pages, plus a live `GET /api/v1/models`:
31
+
32
+ - Requests are `POST https://openrouter.ai/api/v1/chat/completions` with `Authorization: Bearer`. That matches `configs/providers.yaml` `endpoint_path: /chat/completions`.
33
+ - The model field is the slug the visitor selected, including a `:nitro` or `:floor` suffix. Those two suffixes are routing variants, not catalog rows. `:free` and `:batch` are catalog rows and are not rewritten.
34
+ - `pricing.prompt` and `pricing.completion` are USD per token, returned as strings. The live catalog had no `pricing.request` field. A negative price is treated as unknown.
35
+ - The model list answers without a key. If the typed key is rejected, the loader retries the public list.
36
+ - Default routing load-balances providers and falls back to another provider of the same slug. Unsupported sampling parameters are ignored unless `require_parameters` is set. This app does not send a `provider` object, and the estimate says so.
37
+ - A `:free` slug is capped at 20 requests per minute, and at 50 per day until the account has purchased at least 10 credits, then 1,000 per day. Paid slugs are limited by credits, the in-flight spending budget, the upstream provider, and DDoS protection.
38
+ - `HTTP-Referer` and `X-OpenRouter-Title` are optional. They attribute the app on OpenRouter rankings. They are not required for the call, and the client does not add them.
39
+
40
+ ## What the visitor can change
41
+
42
+ - OpenRouter key and model id
43
+ - modules, source files, dimensions, and `max_samples`
44
+ - concurrency from 1 to 100 (the hosted default is 8; `configs/app.yaml` is 100)
45
+
46
+ Temperature, max tokens, streaming, provider kwargs, prompts, and judges are not form fields.
47
+
48
+ ## Key handling
49
+
50
+ The key is held in `ELBENCH_VISITOR_OPENROUTER_API_KEY` for the duration of the estimate or run, then removed. It is not written to `configs/` or `key.txt`. The basic-education executor does write it into `outputs/basic_education/<run_id>/.../basic_education_runtime.yaml`, because that runtime reads the key from the rendered config. The app deletes that run directory when the run returns, including on failure. A killed process can leave the file on disk.
51
+
52
+ ## Local use
53
+
54
+ ```bash
55
+ pip install -e ".[basic-education,dashboard]"
56
+ python dashboard/app.py
57
+ ```
dashboard/app.py ADDED
@@ -0,0 +1,1405 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ELBench-live: the published ELBench results, and a hosted runner for any OpenRouter model.
2
+
3
+ Runs go through ``BenchmarkRunner``, ``OpenAICompatibleClient``, and ``JudgeRouter``, so
4
+ prompts, retries, and scoring are the benchmark's own.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import sys
10
+ import threading
11
+ import time
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ try:
16
+ import spaces # type: ignore[import-not-found]
17
+ except ImportError:
18
+ spaces = None # type: ignore[assignment]
19
+
20
+ ROOT = Path(__file__).resolve().parents[1]
21
+ SRC = ROOT / "src"
22
+ if str(SRC) not in sys.path:
23
+ sys.path.insert(0, str(SRC))
24
+
25
+ import gradio as gr
26
+ import httpx
27
+
28
+ from elbench.dashboard.catalog import Catalog, CatalogFile, load_catalog
29
+ from elbench.dashboard.english import AREA_LABELS, area_label, english_dimension, plain_file
30
+ from elbench.dashboard.leaderboard import (
31
+ LeaderboardStore,
32
+ build_comparison_choices,
33
+ build_comparison_table,
34
+ build_pareto_frontier,
35
+ custom_leaderboard_rows,
36
+ load_official_models,
37
+ official_leaderboard_rows,
38
+ official_per_file_rows,
39
+ pareto_cost_metric_choices,
40
+ pareto_score_metric_choices,
41
+ seed_official_openrouter_prices,
42
+ )
43
+ from elbench.dashboard.model_sizes import ModelSize, load_sizes, model_size
44
+ from elbench.dashboard.plan import Plan, build_plan, render_plan
45
+ from elbench.dashboard.protocol import DEFAULT_SYSTEM_ONE_MODELS, catalog_slug
46
+ from elbench.dashboard.run import (
47
+ execute_run,
48
+ failures_progress_path,
49
+ judged_progress_path,
50
+ make_run_id,
51
+ public_error,
52
+ visitor_api_key,
53
+ )
54
+ from elbench.utils.spend import ROLE_LABELS, SpendLedger, SpendLimits
55
+
56
+ OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
57
+ OPENROUTER_DECISION_MODELS_URL = "https://openrouter.ai/api/v1/models?output_modalities=decisions"
58
+ OPENROUTER_KEY_URL = "https://openrouter.ai/api/v1/key"
59
+ OPENROUTER_KEYS_PAGE = "https://openrouter.ai/settings/keys"
60
+ DEFAULT_TARGET_SLUG = "openai/gpt-5.4"
61
+ DEFAULT_CHAT_JUDGE_SLUG = "qwen/qwen3-235b-a22b-2507"
62
+ DEFAULT_SYSTEM_ONE_JUDGE_SLUG = "liquid/d1-ultra"
63
+ DEFAULT_MULTITURN_CHAT_SLUG = "liquid/lfm-2.5-24b-a2b"
64
+ DEFAULT_BUDGET_USD = 5.0
65
+
66
+ ELBENCH_PAPER_URL = "https://arxiv.org/abs/2608.09548"
67
+ ELBENCH_GITHUB_URL = "https://github.com/ZeroLoss-Lab/ELBench"
68
+ AI_FOR_EDUCATION_LEADERBOARD_URL = "https://benchmarks.ai-for-education.org/"
69
+ ELBENCH_DATASET_URL = "https://huggingface.co/datasets/ZeroLoss-Lab/ELBench"
70
+ ELBENCH_RESULTS_URL = "https://huggingface.co/datasets/ZeroLoss-Lab/ELBench-results"
71
+
72
+ PRESET_QUICK = "quick"
73
+ PRESET_AUTO = "auto"
74
+ PRESET_FULL = "full"
75
+ QUICK_CHECK_ITEMS = 5
76
+
77
+ # Filter sliders: the right-hand end means "no limit".
78
+ PRICE_FILTER_MAX = 30.0
79
+ SIZE_FILTER_MAX = 1000.0
80
+ MODEL_SIZES = load_sizes()
81
+
82
+ FOCUS_CHOICES = [(label, key) for label, key in pareto_score_metric_choices() if not key.startswith("file:")]
83
+ TASK_CHOICES = [("None, use the area above", "")] + [
84
+ (label, key) for label, key in pareto_score_metric_choices() if key.startswith("file:")
85
+ ]
86
+
87
+
88
+ def _load_catalog() -> Catalog | None:
89
+ try:
90
+ return load_catalog(ROOT / "configs")
91
+ except Exception as exc: # noqa: BLE001
92
+ _load_catalog.error = str(exc) # type: ignore[attr-defined]
93
+ return None
94
+
95
+
96
+ _load_catalog.error = "" # type: ignore[attr-defined]
97
+ CATALOG = _load_catalog()
98
+ LEADERBOARD_STORE = LeaderboardStore(ROOT)
99
+ try:
100
+ LEADERBOARD_STORE.sync_from_hub()
101
+ except Exception: # noqa: BLE001
102
+ pass
103
+
104
+
105
+ # ---------------------------------------------------------------------------
106
+ # OpenRouter catalog and prices (one cached snapshot per process)
107
+ # ---------------------------------------------------------------------------
108
+
109
+ CATALOG_SNAPSHOT_TTL_SECONDS = 6 * 3600
110
+ _SNAPSHOT: dict[str, Any] = {"value": None, "fetched_at": 0.0}
111
+ _SNAPSHOT_LOCK = threading.Lock()
112
+
113
+
114
+ def _nonnegative_price(value) -> float | None:
115
+ if value is None or value == "":
116
+ return None
117
+ try:
118
+ price = float(value)
119
+ except (TypeError, ValueError):
120
+ return None
121
+ if price < 0:
122
+ return None
123
+ return price
124
+
125
+
126
+ def _fetch_model_catalog(api_key: str, url: str = OPENROUTER_MODELS_URL, timeout: float = 45) -> dict:
127
+ headers = {"Accept": "application/json"}
128
+ secret = (api_key or "").strip()
129
+ if secret:
130
+ headers["Authorization"] = f"Bearer {secret}"
131
+ response = httpx.get(url, headers=headers, timeout=timeout)
132
+ response.raise_for_status()
133
+ payload = response.json()
134
+ if not isinstance(payload, dict):
135
+ raise ValueError("OpenRouter /api/v1/models did not return an object.")
136
+ return payload
137
+
138
+
139
+ def _per_million(price: float | None) -> str:
140
+ if price is None:
141
+ return "?"
142
+ value = price * 1_000_000
143
+ return f"${value:.2f}" if value >= 0.1 or value == 0 else f"${value:.3f}"
144
+
145
+
146
+ def _parse_catalog_rows(payload: dict) -> tuple[list[dict[str, Any]], dict[str, tuple[float | None, float | None]]]:
147
+ models: list[dict[str, Any]] = []
148
+ prices: dict[str, tuple[float | None, float | None]] = {}
149
+ for item in payload.get("data") or []:
150
+ model_id = str(item.get("id") or "").strip()
151
+ if not model_id:
152
+ continue
153
+ pricing = item.get("pricing") or {}
154
+ prompt = _nonnegative_price(pricing.get("prompt"))
155
+ completion = _nonnegative_price(pricing.get("completion"))
156
+ prices[model_id] = (prompt, completion)
157
+ models.append(
158
+ {
159
+ "id": model_id,
160
+ "name": str(item.get("name") or model_id),
161
+ "prompt": prompt,
162
+ "completion": completion,
163
+ "hugging_face_id": item.get("hugging_face_id") or None,
164
+ }
165
+ )
166
+ models.sort(key=lambda m: m["name"].lower())
167
+ return models, prices
168
+
169
+
170
+ def _model_choices(models: list[dict[str, Any]]) -> list[tuple[str, str]]:
171
+ choices = []
172
+ for m in models:
173
+ if m["prompt"] is None:
174
+ price = "no price listed"
175
+ else:
176
+ price = f"{_per_million(m['prompt'])} in / {_per_million(m['completion'])} out per 1M tokens"
177
+ size = model_size(m["id"], m.get("hugging_face_id"), MODEL_SIZES)
178
+ choices.append((f"{m['name']} · {price}" + (f" · {size.label()}" if size else ""), m["id"]))
179
+ return choices
180
+
181
+
182
+ def _blended_per_1m(prompt: float | None, completion: float | None) -> float | None:
183
+ """The chart's default price measure: 3 parts input to 1 part output, per million tokens."""
184
+ if prompt is None or completion is None:
185
+ return None
186
+ return (3 * prompt + completion) / 4 * 1_000_000
187
+
188
+
189
+ def passes_filters(
190
+ price_per_1m: float | None, size: ModelSize | None, max_price, max_size, include_undisclosed
191
+ ) -> bool:
192
+ if max_price is not None and float(max_price) < PRICE_FILTER_MAX:
193
+ if price_per_1m is None or price_per_1m > float(max_price):
194
+ return False
195
+ if size is None:
196
+ return bool(include_undisclosed)
197
+ return max_size is None or float(max_size) >= SIZE_FILTER_MAX or size.total_b <= float(max_size)
198
+
199
+
200
+ def _size_of(slug: str, snapshot: dict[str, Any]) -> ModelSize | None:
201
+ hf_ids = snapshot.get("hf_ids") or {}
202
+ return model_size(slug, hf_ids.get(slug) or hf_ids.get(catalog_slug(slug)), MODEL_SIZES)
203
+
204
+
205
+ def _filtered_chat_models(snapshot: dict[str, Any], max_price, max_size, include_undisclosed) -> list[dict[str, Any]]:
206
+ return [
207
+ m
208
+ for m in snapshot["chat_models"]
209
+ if passes_filters(
210
+ _blended_per_1m(m["prompt"], m["completion"]),
211
+ model_size(m["id"], m.get("hugging_face_id"), MODEL_SIZES),
212
+ max_price,
213
+ max_size,
214
+ include_undisclosed,
215
+ )
216
+ ]
217
+
218
+
219
+ def _default_system_one_models() -> list[dict[str, Any]]:
220
+ return [
221
+ {"id": slug, "name": name, "prompt": price, "completion": 0.0, "hugging_face_id": None}
222
+ for name, slug, price in DEFAULT_SYSTEM_ONE_MODELS
223
+ ]
224
+
225
+
226
+ def _default_prices() -> dict[str, tuple[float | None, float | None]]:
227
+ table: dict[str, tuple[float | None, float | None]] = seed_official_openrouter_prices()
228
+ for _, slug, prompt_price in DEFAULT_SYSTEM_ONE_MODELS:
229
+ table[slug] = (prompt_price, 0.0)
230
+ return table
231
+
232
+
233
+ def openrouter_snapshot() -> dict[str, Any]:
234
+ """Chat models, decision models, and prices from OpenRouter, cached for a few hours."""
235
+ with _SNAPSHOT_LOCK:
236
+ cached = _SNAPSHOT["value"]
237
+ if cached is not None and time.time() - _SNAPSHOT["fetched_at"] < CATALOG_SNAPSHOT_TTL_SECONDS:
238
+ return cached
239
+ prices = _default_prices()
240
+ try:
241
+ chat_models, live_prices = _parse_catalog_rows(_fetch_model_catalog("", OPENROUTER_MODELS_URL, timeout=15))
242
+ except Exception: # noqa: BLE001
243
+ if cached is not None:
244
+ return cached
245
+ return {
246
+ "hf_ids": {},
247
+ "chat_models": [],
248
+ "decision_models": _default_system_one_models(),
249
+ "prices": prices,
250
+ "label": "Prices stored with the published results (OpenRouter could not be reached).",
251
+ "live": False,
252
+ }
253
+ prices.update(live_prices)
254
+ decision_models = _default_system_one_models()
255
+ try:
256
+ fetched, decision_prices = _parse_catalog_rows(
257
+ _fetch_model_catalog("", OPENROUTER_DECISION_MODELS_URL, timeout=15)
258
+ )
259
+ if fetched:
260
+ decision_models = fetched
261
+ prices.update(decision_prices)
262
+ except Exception: # noqa: BLE001
263
+ pass
264
+ value = {
265
+ "hf_ids": {m["id"]: m["hugging_face_id"] for m in chat_models if m["hugging_face_id"]},
266
+ "chat_models": chat_models,
267
+ "decision_models": decision_models,
268
+ "prices": prices,
269
+ "label": f"Prices from OpenRouter, {time.strftime('%d %b %Y %H:%M UTC', time.gmtime())}.",
270
+ "live": True,
271
+ }
272
+ _SNAPSHOT.update(value=value, fetched_at=time.time())
273
+ return value
274
+
275
+
276
+ def price_snapshot() -> tuple[dict[str, tuple[float | None, float | None]], str]:
277
+ snapshot = openrouter_snapshot()
278
+ return dict(snapshot["prices"]), snapshot["label"]
279
+
280
+
281
+ def _price_pair(prices: dict, model_id: str) -> tuple[float | None, float | None]:
282
+ key = str(model_id or "").strip()
283
+ raw = (prices or {}).get(key) or (prices or {}).get(catalog_slug(key))
284
+ if not raw:
285
+ return None, None
286
+ return raw[0], raw[1]
287
+
288
+
289
+ def _pick_preferred(choices: list[tuple[str, str]], preferred: str) -> str:
290
+ slugs = [slug for _, slug in choices]
291
+ if preferred in slugs or not slugs:
292
+ return preferred
293
+ return slugs[0]
294
+
295
+
296
+ def check_key(api_key: str) -> tuple[bool, str]:
297
+ """Free call to OpenRouter's key endpoint: is the key valid, and how much credit is left."""
298
+ try:
299
+ response = httpx.get(OPENROUTER_KEY_URL, headers={"Authorization": f"Bearer {api_key}"}, timeout=15)
300
+ except Exception: # noqa: BLE001
301
+ return True, "Could not check the key with OpenRouter just now; the run will tell you if it is rejected."
302
+ if response.status_code in {401, 403}:
303
+ return False, "OpenRouter did not accept this API key. Check that it was copied in full."
304
+ if response.status_code != 200:
305
+ return True, "Could not check the key with OpenRouter just now; the run will tell you if it is rejected."
306
+ data = (response.json() or {}).get("data") or {}
307
+ remaining = data.get("limit_remaining")
308
+ if isinstance(remaining, (int, float)):
309
+ return True, f"Key accepted. This key has ${remaining:,.2f} of credit left."
310
+ return True, "Key accepted."
311
+
312
+
313
+ # ---------------------------------------------------------------------------
314
+ # Run scope
315
+ # ---------------------------------------------------------------------------
316
+
317
+
318
+ def _module_choices() -> list[tuple[str, str]]:
319
+ if CATALOG is None:
320
+ return []
321
+ return [(area_label(m), m) for m in CATALOG.modules()]
322
+
323
+
324
+ def _file_label(entry: CatalogFile) -> str:
325
+ grading = "graded by the judge model" if entry.scoring_mode in {"llm", "runtime"} else "checked automatically"
326
+ return f"{area_label(entry.module)} · {plain_file(entry.canonical_name)} · {entry.item_count} questions, {grading}"
327
+
328
+
329
+ def _file_choices(files: list[CatalogFile]) -> list[tuple[str, str]]:
330
+ return [(_file_label(entry), entry.canonical_name) for entry in files]
331
+
332
+
333
+ def _dimension_choices(files: list[CatalogFile]) -> list[tuple[str, str]]:
334
+ seen: list[str] = []
335
+ for entry in files:
336
+ for dimension in entry.dimensions:
337
+ if dimension not in seen:
338
+ seen.append(dimension)
339
+ return [(english_dimension(d), d) for d in seen]
340
+
341
+
342
+ def _selected_entries(names: list[str] | None) -> list[CatalogFile]:
343
+ if CATALOG is None:
344
+ return []
345
+ wanted = set(names or [])
346
+ return [entry for entry in CATALOG.files if entry.canonical_name in wanted]
347
+
348
+
349
+ def _preset_choices() -> list[tuple[str, str]]:
350
+ if CATALOG is None:
351
+ return []
352
+ auto_items = sum(entry.item_count for entry in CATALOG.default_files())
353
+ full_items = sum(entry.item_count for entry in CATALOG.files)
354
+ return [
355
+ (f"Quick check · {QUICK_CHECK_ITEMS} questions, to see that everything works", PRESET_QUICK),
356
+ (f"Automatically checked set · {auto_items:,} questions, no judge model needed", PRESET_AUTO),
357
+ (f"Full benchmark · all {full_items:,} questions · appears on the leaderboard chart", PRESET_FULL),
358
+ ]
359
+
360
+
361
+ def apply_preset(preset: str):
362
+ if CATALOG is None:
363
+ return gr.update(), gr.update(), gr.update(), gr.update()
364
+ if preset == PRESET_FULL:
365
+ files = list(CATALOG.files)
366
+ modules = CATALOG.modules()
367
+ cap = None
368
+ else:
369
+ files = CATALOG.default_files()
370
+ modules = CATALOG.default_modules()
371
+ cap = QUICK_CHECK_ITEMS if preset == PRESET_QUICK else None
372
+ return (
373
+ gr.update(value=modules),
374
+ gr.update(value=[entry.canonical_name for entry in files]),
375
+ gr.update(choices=_dimension_choices(files), value=[]),
376
+ gr.update(value=cap),
377
+ )
378
+
379
+
380
+ def on_modules(modules: list[str]):
381
+ if CATALOG is None:
382
+ return gr.update(), gr.update()
383
+ chosen = CATALOG.files_in_modules(set(modules or []))
384
+ return (
385
+ gr.update(value=[entry.canonical_name for entry in chosen]),
386
+ gr.update(choices=_dimension_choices(chosen), value=[]),
387
+ )
388
+
389
+
390
+ def on_files(files: list[str]):
391
+ return gr.update(choices=_dimension_choices(_selected_entries(files)), value=[])
392
+
393
+
394
+ def _max_samples(value) -> int | None:
395
+ if value is None or value == "":
396
+ return None
397
+ return int(value)
398
+
399
+
400
+ def _budget(value) -> float | None:
401
+ if value is None or value == "":
402
+ return None
403
+ try:
404
+ number = float(value)
405
+ except (TypeError, ValueError):
406
+ return None
407
+ return number if number > 0 else None
408
+
409
+
410
+ # ---------------------------------------------------------------------------
411
+ # Model pickers
412
+ # ---------------------------------------------------------------------------
413
+
414
+
415
+ def _models_shown_note(shown: int, total: int) -> str:
416
+ return f"Showing {shown:,} of {total:,} OpenRouter models." if total else ""
417
+
418
+
419
+ def load_model_pickers(use_system_one: bool, max_price, max_size, include_undisclosed):
420
+ snapshot = openrouter_snapshot()
421
+ chat = _model_choices(snapshot["chat_models"])
422
+ targets = _model_choices(_filtered_chat_models(snapshot, max_price, max_size, include_undisclosed))
423
+ judge_choices = _model_choices(snapshot["decision_models"]) if use_system_one else chat
424
+ judge_default = DEFAULT_SYSTEM_ONE_JUDGE_SLUG if use_system_one else DEFAULT_CHAT_JUDGE_SLUG
425
+ return (
426
+ gr.update(choices=targets, value=_pick_preferred(targets, DEFAULT_TARGET_SLUG)),
427
+ gr.update(choices=judge_choices, value=_pick_preferred(judge_choices, judge_default)),
428
+ gr.update(choices=chat, value=_pick_preferred(chat, DEFAULT_MULTITURN_CHAT_SLUG)),
429
+ dict(snapshot["prices"]),
430
+ _models_shown_note(len(targets), len(chat)),
431
+ )
432
+
433
+
434
+ def filter_target_models(current, max_price, max_size, include_undisclosed):
435
+ snapshot = openrouter_snapshot()
436
+ targets = _model_choices(_filtered_chat_models(snapshot, max_price, max_size, include_undisclosed))
437
+ slugs = [slug for _, slug in targets]
438
+ value = current if current in slugs else None
439
+ return (
440
+ gr.update(choices=targets, value=value),
441
+ _models_shown_note(len(targets), len(snapshot["chat_models"])),
442
+ )
443
+
444
+
445
+ def on_toggle_system_one(use_system_one: bool):
446
+ snapshot = openrouter_snapshot()
447
+ if use_system_one:
448
+ choices = _model_choices(snapshot["decision_models"])
449
+ return gr.update(
450
+ label="Judge (decision model)",
451
+ choices=choices,
452
+ value=_pick_preferred(choices, DEFAULT_SYSTEM_ONE_JUDGE_SLUG),
453
+ info="Decision models return a verdict with no written output, which is much cheaper per answer.",
454
+ )
455
+ choices = _model_choices(snapshot["chat_models"])
456
+ return gr.update(
457
+ label="Judge model",
458
+ choices=choices,
459
+ value=_pick_preferred(choices, DEFAULT_CHAT_JUDGE_SLUG),
460
+ info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's judge.",
461
+ )
462
+
463
+
464
+ def on_toggle_multiturn(use_multiturn: bool):
465
+ return gr.update(visible=bool(use_multiturn), interactive=bool(use_multiturn))
466
+
467
+
468
+ # ---------------------------------------------------------------------------
469
+ # Estimate and run
470
+ # ---------------------------------------------------------------------------
471
+
472
+
473
+ def _usd(value: float | None) -> str:
474
+ if value is None:
475
+ return "?"
476
+ if value >= 0.995:
477
+ return f"${value:,.2f}"
478
+ if value >= 0.01:
479
+ return f"${value:.2f}"
480
+ if value == 0:
481
+ return "$0"
482
+ return "$" + f"{value:.6f}".rstrip("0")
483
+
484
+
485
+ def _make_plan(
486
+ *,
487
+ model_id,
488
+ judge_model_id,
489
+ use_system_one_judge,
490
+ use_multiturn_chat,
491
+ multiturn_model_id,
492
+ modules,
493
+ files,
494
+ dimensions,
495
+ max_samples,
496
+ concurrency,
497
+ prices,
498
+ ) -> Plan:
499
+ prompt_price, completion_price = _price_pair(prices, str(model_id).strip())
500
+ judge_prompt_price, judge_completion_price = _price_pair(prices, str(judge_model_id).strip())
501
+ mt_prompt_price, mt_completion_price = _price_pair(prices, str(multiturn_model_id or "").strip())
502
+ return build_plan(
503
+ CATALOG,
504
+ openrouter_model=str(model_id).strip(),
505
+ modules=set(modules or []),
506
+ source_files=set(files or []),
507
+ dimensions=set(dimensions or []),
508
+ max_samples=_max_samples(max_samples),
509
+ prompt_usd_per_token=prompt_price,
510
+ completion_usd_per_token=completion_price,
511
+ max_concurrency=int(concurrency),
512
+ judge_openrouter_model=str(judge_model_id).strip(),
513
+ use_system_one_judge=bool(use_system_one_judge),
514
+ judge_prompt_usd_per_token=judge_prompt_price,
515
+ judge_completion_usd_per_token=judge_completion_price,
516
+ use_multiturn_chat_model=bool(use_multiturn_chat),
517
+ multiturn_openrouter_model=str(multiturn_model_id or "").strip() if use_multiturn_chat else None,
518
+ multiturn_prompt_usd_per_token=mt_prompt_price,
519
+ multiturn_completion_usd_per_token=mt_completion_price,
520
+ )
521
+
522
+
523
+ def budget_problems(plan: Plan, total: float | None, per_role: dict[str, float | None]) -> list[str]:
524
+ """Limits that the lowest estimate already exceeds, so the run could not finish."""
525
+ problems: list[str] = []
526
+ if total is not None and plan.low_usd is not None and plan.low_usd > total:
527
+ problems.append(
528
+ f"Your spending limit ({_usd(total)}) is below the lowest estimate for this run "
529
+ f"({_usd(plan.low_usd)}). Raise the limit or choose fewer questions."
530
+ )
531
+ for role, cap in per_role.items():
532
+ low = (plan.component_usd.get(role) or (None, None))[0]
533
+ if cap is not None and low is not None and low > cap:
534
+ problems.append(
535
+ f"The limit for the {ROLE_LABELS[role]} ({_usd(cap)}) is below its lowest estimate ({_usd(low)})."
536
+ )
537
+ return problems
538
+
539
+
540
+ def plain_estimate(
541
+ plan: Plan,
542
+ *,
543
+ total_budget: float | None,
544
+ role_budgets: dict[str, float | None],
545
+ key_note: str,
546
+ resumable: dict | None,
547
+ ) -> str:
548
+ lines: list[str] = [f"*{key_note}*", ""] if key_note else []
549
+ errors = plan.error_messages()
550
+ if errors:
551
+ lines.append("**This run can't start yet:**")
552
+ lines.extend(f"- {message}" for message in errors)
553
+ return "\n".join(lines)
554
+
555
+ questions = plan.standard_requests + plan.basic_tasks
556
+ lines.append(f"### {questions:,} questions for `{plan.protocol.openrouter_model}`")
557
+ graded = plan.judge_calls + plan.basic_tasks
558
+ if graded and plan.judge_model_config is not None:
559
+ lines.append(
560
+ f"{graded:,} answers are graded by `{plan.judge_model_config.model_name}`; "
561
+ "the rest are checked automatically against the answer key."
562
+ )
563
+ else:
564
+ lines.append("Every answer is checked automatically against the answer key.")
565
+ lines.append("")
566
+
567
+ if plan.low_usd is None or plan.high_usd is None:
568
+ lines.append(
569
+ "**No cost estimate:** OpenRouter lists no price for this model. To run it anyway, tick "
570
+ "*Run even though this model has no listed price* under Advanced settings."
571
+ )
572
+ else:
573
+ lines.append(
574
+ f"**Estimated cost: {_usd(plan.low_usd)} to {_usd(plan.high_usd)}.** The low end counts the "
575
+ "questions only; the high end assumes every answer runs to the length limit, so real runs "
576
+ "land well below it."
577
+ )
578
+ for role in ("target", "judge"):
579
+ if role in plan.component_usd:
580
+ low, high = plan.component_usd[role]
581
+ cap = role_budgets.get(role)
582
+ limit = f" · limit {_usd(cap)}" if cap is not None else ""
583
+ lines.append(f"- {ROLE_LABELS[role].capitalize()}: {_usd(low)} to {_usd(high)}{limit}")
584
+ if role_budgets.get("student") is not None:
585
+ lines.append(f"- Simulated student: not estimated · limit {_usd(role_budgets['student'])}")
586
+ if plan.basic_tasks:
587
+ lines.append(
588
+ "- Teaching tasks are multi-turn conversations whose length isn't known in advance; "
589
+ "their judge and simulated-student calls are not in this estimate but do count toward your limits."
590
+ )
591
+ lines.append("")
592
+
593
+ problems = budget_problems(plan, total_budget, role_budgets)
594
+ if problems:
595
+ lines.append("**Spending limit too low:**")
596
+ lines.extend(f"- {problem}" for problem in problems)
597
+ elif total_budget is None:
598
+ lines.append("No spending limit is set. The run continues until it finishes or your key runs out of credit.")
599
+ elif plan.high_usd is not None and plan.high_usd <= total_budget:
600
+ lines.append(f"Your limit of {_usd(total_budget)} covers even the high estimate.")
601
+ else:
602
+ lines.append(
603
+ f"**Spending limit {_usd(total_budget)}.** If spending reaches it, the run pauses and keeps every "
604
+ "finished answer. Press **Run** again with a higher limit to carry on from where it stopped."
605
+ )
606
+ if total_budget is not None or any(cap is not None for cap in role_budgets.values()):
607
+ lines.append(
608
+ "Questions already sent when a limit is reached still finish, so the final total can go slightly over it."
609
+ )
610
+ lines.append("")
611
+
612
+ if plan.is_full_benchmark:
613
+ lines.append("This is a full run: once it finishes, it appears on the leaderboard chart next to the published models.")
614
+ else:
615
+ lines.append(
616
+ "This is a partial run. Its results are saved with the visitor runs, but the chart only shows "
617
+ "models scored on every question in an area, so the comparison stays fair."
618
+ )
619
+ if plan.protocol.openrouter_model.endswith(":free"):
620
+ lines.append("Free models on OpenRouter are limited to about 20 requests a minute, so this will be slow.")
621
+ if resumable is not None:
622
+ lines.append(
623
+ f"\nAn earlier run with these exact settings stopped after {resumable.get('total_judged', 0)} "
624
+ "answers. Running again picks up where it left off."
625
+ )
626
+ return "\n".join(lines)
627
+
628
+
629
+ def _role_budgets(budget_target, budget_judge, budget_student) -> dict[str, float | None]:
630
+ return {"target": _budget(budget_target), "judge": _budget(budget_judge), "student": _budget(budget_student)}
631
+
632
+
633
+ def estimate(
634
+ api_key,
635
+ model_id,
636
+ judge_model_id,
637
+ use_system_one_judge,
638
+ use_multiturn_chat,
639
+ multiturn_model_id,
640
+ modules,
641
+ files,
642
+ dimensions,
643
+ max_samples,
644
+ concurrency,
645
+ prices,
646
+ total_budget,
647
+ budget_target,
648
+ budget_judge,
649
+ budget_student,
650
+ ):
651
+ if CATALOG is None:
652
+ return f"The benchmark data did not load: {_load_catalog.error}", ""
653
+ secret = (api_key or "").strip()
654
+ if not secret:
655
+ return f"Paste your OpenRouter API key first. You can create one at [openrouter.ai]({OPENROUTER_KEYS_PAGE}).", ""
656
+ if not str(model_id or "").strip():
657
+ return "Choose the model you want to test.", ""
658
+ if not str(judge_model_id or "").strip():
659
+ return "Choose a judge model under Advanced settings.", ""
660
+ key_ok, key_note = check_key(secret)
661
+ if not key_ok:
662
+ return key_note, ""
663
+ try:
664
+ with visitor_api_key(secret):
665
+ plan = _make_plan(
666
+ model_id=model_id,
667
+ judge_model_id=judge_model_id,
668
+ use_system_one_judge=use_system_one_judge,
669
+ use_multiturn_chat=use_multiturn_chat,
670
+ multiturn_model_id=multiturn_model_id,
671
+ modules=modules,
672
+ files=files,
673
+ dimensions=dimensions,
674
+ max_samples=max_samples,
675
+ concurrency=concurrency,
676
+ prices=prices,
677
+ )
678
+ except Exception as exc: # noqa: BLE001
679
+ return public_error(exc, secret), ""
680
+ resumable = LEADERBOARD_STORE.find_resumable_run(config_hash=plan.config_hash)
681
+ summary = plain_estimate(
682
+ plan,
683
+ total_budget=_budget(total_budget),
684
+ role_budgets=_role_budgets(budget_target, budget_judge, budget_student),
685
+ key_note=key_note,
686
+ resumable=resumable,
687
+ )
688
+ return summary, render_plan(plan)
689
+
690
+
691
+ def _spent_so_far(run_id: str) -> float:
692
+ if CATALOG is None or not run_id:
693
+ return 0.0
694
+ path = CATALOG.config.project_root / "outputs" / "hosted" / run_id / "spend.jsonl"
695
+ if not path.exists():
696
+ return 0.0
697
+ return sum(SpendLedger(path, SpendLimits()).spent().values())
698
+
699
+
700
+ def _count_lines(path: Path) -> int:
701
+ if not path.exists():
702
+ return 0
703
+ with path.open("r", encoding="utf-8", errors="replace") as handle:
704
+ return sum(1 for _ in handle)
705
+
706
+
707
+ def _area_scores_line(norm: dict) -> str:
708
+ parts = []
709
+ for module, key in (("通用模型", "general_pct"), ("安全可信", "safety_pct"), ("基本教育", "basic_education_pct"), ("高阶育人", "high_level_pct")):
710
+ value = norm.get(key)
711
+ if isinstance(value, (int, float)):
712
+ parts.append(f"{area_label(module)} **{value:.1f}**")
713
+ return " · ".join(parts)
714
+
715
+
716
+ def run_evaluation(
717
+ api_key,
718
+ model_id,
719
+ judge_model_id,
720
+ use_system_one_judge,
721
+ use_multiturn_chat,
722
+ multiturn_model_id,
723
+ modules,
724
+ files,
725
+ dimensions,
726
+ max_samples,
727
+ concurrency,
728
+ prices,
729
+ total_budget,
730
+ budget_target,
731
+ budget_judge,
732
+ budget_student,
733
+ confirmed,
734
+ accept_unpriced,
735
+ resume_previous=True,
736
+ persist_to_leaderboard=True,
737
+ ):
738
+ empty_table: list[list[Any]] = []
739
+ if CATALOG is None:
740
+ yield f"The benchmark data did not load: {_load_catalog.error}", empty_table, None
741
+ return
742
+ secret = (api_key or "").strip()
743
+ if not secret:
744
+ yield "Paste your OpenRouter API key first.", empty_table, None
745
+ return
746
+ if not confirmed:
747
+ yield "Tick the box above to confirm the run can be charged to your OpenRouter key.", empty_table, None
748
+ return
749
+ if not str(model_id or "").strip():
750
+ yield "Choose the model you want to test.", empty_table, None
751
+ return
752
+ if not str(judge_model_id or "").strip():
753
+ yield "Choose a judge model under Advanced settings.", empty_table, None
754
+ return
755
+ try:
756
+ sample_cap = _max_samples(max_samples)
757
+ worker_concurrency = int(concurrency)
758
+ except (TypeError, ValueError) as exc:
759
+ yield public_error(exc, secret), empty_table, None
760
+ return
761
+ total_limit = _budget(total_budget)
762
+ role_limits = _role_budgets(budget_target, budget_judge, budget_student)
763
+
764
+ holder: dict[str, object] = {"status": "starting", "done": False, "error": "", "result": None, "run_id": ""}
765
+ model_holder: dict[str, str] = {}
766
+
767
+ def _worker() -> None:
768
+ try:
769
+ with visitor_api_key(secret):
770
+ plan = _make_plan(
771
+ model_id=model_id,
772
+ judge_model_id=judge_model_id,
773
+ use_system_one_judge=use_system_one_judge,
774
+ use_multiturn_chat=use_multiturn_chat,
775
+ multiturn_model_id=multiturn_model_id,
776
+ modules=modules,
777
+ files=files,
778
+ dimensions=dimensions,
779
+ max_samples=sample_cap,
780
+ concurrency=worker_concurrency,
781
+ prices=prices,
782
+ )
783
+ model_holder["model_id"] = plan.protocol.model_config.model_id
784
+ problems = plan.error_messages() + budget_problems(plan, total_limit, role_limits)
785
+ if plan.needs_price_ack and not accept_unpriced:
786
+ problems.append(
787
+ "OpenRouter lists no price for this model. Tick *Run even though this model has no "
788
+ "listed price* under Advanced settings to run it anyway."
789
+ )
790
+ if problems:
791
+ holder["error"] = "**This run can't start yet:**\n" + "\n".join(f"- {p}" for p in problems)
792
+ return
793
+ resumable = LEADERBOARD_STORE.find_resumable_run(config_hash=plan.config_hash) if resume_previous else None
794
+ active_run_id = (
795
+ str(resumable["run_id"])
796
+ if resumable is not None and resumable.get("run_id")
797
+ else make_run_id(plan.config_hash)
798
+ )
799
+ holder["run_id"] = active_run_id
800
+ holder["status"] = "Continuing" if resumable is not None else "Running"
801
+ holder["result"] = execute_run(
802
+ CATALOG,
803
+ plan,
804
+ max_concurrency=worker_concurrency,
805
+ accept_unpriced=bool(accept_unpriced),
806
+ run_id=active_run_id,
807
+ resume_previous=bool(resume_previous),
808
+ leaderboard_store=LEADERBOARD_STORE,
809
+ persist_to_leaderboard=bool(persist_to_leaderboard),
810
+ total_budget_usd=total_limit,
811
+ role_budgets_usd=role_limits,
812
+ )
813
+ except Exception as exc: # noqa: BLE001
814
+ holder["error"] = public_error(exc, secret)
815
+ finally:
816
+ holder["done"] = True
817
+
818
+ thread = threading.Thread(target=_worker, daemon=True)
819
+ thread.start()
820
+ while not holder["done"]:
821
+ model_key = model_holder.get("model_id", "")
822
+ active_id = str(holder.get("run_id") or "")
823
+ judged = failed = 0
824
+ if model_key and active_id:
825
+ judged = _count_lines(judged_progress_path(CATALOG.config.project_root, active_id, model_key))
826
+ failed = _count_lines(failures_progress_path(CATALOG.config.project_root, active_id, model_key))
827
+ spent = _spent_so_far(active_id)
828
+ limit = f" of your {_usd(total_limit)} limit" if total_limit is not None else ""
829
+ yield (
830
+ f"**{holder['status']}…** {judged:,} answers scored"
831
+ + (f", {failed:,} hit errors" if failed else "")
832
+ + f". Spent so far: {_usd(spent)}{limit}. Every finished answer is saved as it completes.",
833
+ empty_table,
834
+ None,
835
+ )
836
+ time.sleep(1)
837
+ thread.join(timeout=1)
838
+
839
+ if holder.get("error"):
840
+ yield str(holder["error"]), empty_table, None
841
+ return
842
+ result = holder.get("result") or {}
843
+ summary = result.get("summary") or {}
844
+ norm = result.get("normalized_scores") or {}
845
+ judged_total = int(summary.get("total_judged") or 0)
846
+ failures = int(summary.get("total_failures") or 0)
847
+ status_label = str(result.get("status") or "completed")
848
+ spent_text = _usd(float(result.get("spent_usd") or 0.0))
849
+
850
+ if result.get("stopped_by_budget"):
851
+ heading = "### Paused at your spending limit"
852
+ body = (
853
+ f"{judged_total:,} answers scored for {spent_text}. Everything finished so far is saved. "
854
+ "Raise the limit and press **Run** again to carry on from here."
855
+ )
856
+ elif status_label == "completed":
857
+ heading = "### Finished"
858
+ body = f"{judged_total:,} answers scored for {spent_text}."
859
+ elif status_label == "interrupted":
860
+ heading = "### Stopped early"
861
+ body = (
862
+ f"{judged_total:,} answers scored for {spent_text} before the run stopped"
863
+ + (f": {result['interrupted_error']}" if result.get("interrupted_error") else ".")
864
+ + " Press **Run** again to continue."
865
+ )
866
+ else:
867
+ heading = "### Finished with some errors"
868
+ body = (
869
+ f"{judged_total:,} answers scored for {spent_text}; {failures:,} hit errors, usually rate limits "
870
+ "or timeouts. Press **Run** again to retry only those."
871
+ )
872
+ scores = _area_scores_line(norm)
873
+ lines = [heading, "", body]
874
+ if scores:
875
+ lines += ["", f"Scores on the questions that ran: {scores}"]
876
+ if persist_to_leaderboard:
877
+ lines += ["", "Saved to the visitor-run leaderboard."]
878
+ if status_label == "completed" and (result.get("leaderboard_record") or {}).get("is_full_benchmark"):
879
+ lines[-1] += " It now appears on the leaderboard chart; press **Refresh** there to see it."
880
+ else:
881
+ lines += ["", "Not saved to the leaderboard (you opted out)."]
882
+ lines += ["", f"<sub>Run id `{result.get('run_id')}`</sub>"]
883
+ yield "\n".join(lines), result.get("summary_rows") or empty_table, result.get("download_paths")
884
+
885
+
886
+ # ---------------------------------------------------------------------------
887
+ # Leaderboard
888
+ # ---------------------------------------------------------------------------
889
+
890
+
891
+ def _metric(focus: str, single_task: str) -> str:
892
+ return single_task or focus or "overall"
893
+
894
+
895
+ def leaderboard_view(
896
+ focus,
897
+ single_task,
898
+ cost_metric,
899
+ include_visitor_runs,
900
+ max_price,
901
+ max_size,
902
+ include_undisclosed,
903
+ prices,
904
+ ):
905
+ runs = LEADERBOARD_STORE.load_runs()
906
+ snapshot = openrouter_snapshot()
907
+ merged, label = price_snapshot()
908
+ if isinstance(prices, dict):
909
+ merged.update(prices)
910
+
911
+ def keep(candidate: dict[str, Any]) -> bool:
912
+ return passes_filters(
913
+ candidate["cost"], _size_of(candidate["slug"], snapshot), max_price, max_size, include_undisclosed
914
+ )
915
+
916
+ fig, rows, md = build_pareto_frontier(
917
+ runs,
918
+ merged,
919
+ score_metric=_metric(focus, single_task),
920
+ cost_metric=cost_metric or "blended_per_1m",
921
+ include_custom_runs=bool(include_visitor_runs),
922
+ keep=keep,
923
+ )
924
+ return fig, rows, f"{md}\n\n<sub>{label}</sub>"
925
+
926
+
927
+ def refresh_leaderboards(
928
+ focus,
929
+ single_task,
930
+ cost_metric,
931
+ include_visitor_runs,
932
+ max_price,
933
+ max_size,
934
+ include_undisclosed,
935
+ primary_sel,
936
+ baseline_sels,
937
+ granularity,
938
+ prices,
939
+ ):
940
+ runs = LEADERBOARD_STORE.sync_from_hub()
941
+ fig, rows, md = leaderboard_view(
942
+ focus, single_task, cost_metric, include_visitor_runs, max_price, max_size, include_undisclosed, prices
943
+ )
944
+ primary_choices, baseline_choices = build_comparison_choices(runs)
945
+ keys = [value for _, value in primary_choices]
946
+ resolved_primary = primary_sel if primary_sel in keys else (keys[0] if keys else None)
947
+ resolved_baselines = [b for b in (baseline_sels or []) if b in keys] or [
948
+ key for key in ("official:deepseek-v4-flash", "official:gpt-5.4") if key in keys and key != resolved_primary
949
+ ]
950
+ headers, comp_rows, comp_note = build_comparison_table(resolved_primary, resolved_baselines, granularity, runs)
951
+ return (
952
+ fig,
953
+ rows,
954
+ md,
955
+ custom_leaderboard_rows(runs),
956
+ f"{len(runs)} visitor run(s). {LEADERBOARD_STORE.sync_status}",
957
+ gr.update(choices=primary_choices, value=resolved_primary),
958
+ gr.update(choices=baseline_choices, value=resolved_baselines),
959
+ gr.Dataframe(headers=headers, value=comp_rows, wrap=True),
960
+ comp_note,
961
+ )
962
+
963
+
964
+ def update_comparison(primary_sel, baseline_sels, granularity):
965
+ headers, comp_rows, comp_note = build_comparison_table(
966
+ primary_sel, baseline_sels, granularity, LEADERBOARD_STORE.load_runs()
967
+ )
968
+ return gr.Dataframe(headers=headers, value=comp_rows, wrap=True), comp_note
969
+
970
+
971
+ def _about_markdown() -> str:
972
+ counts: dict[str, int] = {}
973
+ if CATALOG is not None:
974
+ for entry in CATALOG.files:
975
+ counts[entry.module] = counts.get(entry.module, 0) + entry.item_count
976
+ rows = "\n".join(
977
+ f"| **{label}** | {counts.get(module, 0):,} | {description} |"
978
+ for module, (label, description) in AREA_LABELS.items()
979
+ )
980
+ return f"""## What ELBench measures
981
+
982
+ ELBench asks a model the kinds of things a teacher, tutor or school assistant has to get right, and scores
983
+ the answers in four areas. A single overall number never tells the whole story, so every result is shown
984
+ area by area too.
985
+
986
+ | Area | Questions | What it covers |
987
+ |:-----|:---------:|:---------------|
988
+ {rows}
989
+
990
+ ## How the scores work
991
+
992
+ - Each area is scored from 0 to 100. **Overall** is the average of the four areas.
993
+ - Questions with a single right answer are checked automatically. Open-ended answers (safety guidance,
994
+ teaching, character and thinking skills) are graded by a judge model against a rubric.
995
+ - The published results include a 95% range from 10,000 bootstrap resamples. When two models' ranges
996
+ overlap, the difference between them could disappear on a re-run, and the chart says so.
997
+ - **Best value** models score higher than every model that costs less. Price is the OpenRouter price per
998
+ million tokens, weighting input three times as heavily as output, a common way to compare models.
999
+
1000
+ ## Visitor runs
1001
+
1002
+ - Runs use the same questions, prompts, retries and scoring code as the paper.
1003
+ - Models are called through OpenRouter, which may route a request to any provider that serves the model.
1004
+ - The default judge is Qwen3-235B on OpenRouter. The paper used its own hosted Qwen judge, so judged scores
1005
+ from visitor runs can differ slightly from the published ones.
1006
+ - Only runs that answered every question in an area appear on the chart for that area, so partial runs never
1007
+ compete with complete ones.
1008
+
1009
+ ## Your key and your data
1010
+
1011
+ - Your OpenRouter key is used only while your run is going and is removed from every file the run writes.
1012
+ It is never saved or shown on the leaderboard.
1013
+ - If you leave *Add my results to the public leaderboard* ticked, the model name, settings, scores and
1014
+ answers are published in a public Hugging Face dataset.
1015
+
1016
+ ## Learn more
1017
+
1018
+ - [Paper (arXiv:2608.09548)]({ELBENCH_PAPER_URL}) · [Code]({ELBENCH_GITHUB_URL}) ·
1019
+ [Questions]({ELBENCH_DATASET_URL}) · [Published results and audits]({ELBENCH_RESULTS_URL}) ·
1020
+ [AI for Education leaderboard]({AI_FOR_EDUCATION_LEADERBOARD_URL})
1021
+ """
1022
+
1023
+
1024
+ def _zerogpu_noop() -> str:
1025
+ """No-op handler so Hugging Face ZeroGPU passes its startup check without burning GPU quota."""
1026
+ return "ok"
1027
+
1028
+
1029
+ if spaces is not None and hasattr(spaces, "GPU"):
1030
+ _zerogpu_noop = spaces.GPU(duration=1)(_zerogpu_noop)
1031
+
1032
+
1033
+ def _filter_controls():
1034
+ with gr.Row():
1035
+ max_price = gr.Slider(
1036
+ label="Highest price",
1037
+ minimum=0.1,
1038
+ maximum=PRICE_FILTER_MAX,
1039
+ step=0.1,
1040
+ value=PRICE_FILTER_MAX,
1041
+ info="Dollars per million tokens. All the way right shows every price.",
1042
+ )
1043
+ max_size = gr.Slider(
1044
+ label="Largest model size",
1045
+ minimum=1,
1046
+ maximum=SIZE_FILTER_MAX,
1047
+ step=1,
1048
+ value=SIZE_FILTER_MAX,
1049
+ info="Billions of parameters. All the way right shows every size.",
1050
+ )
1051
+ include_undisclosed = gr.Checkbox(
1052
+ label="Include models that don't publish their size",
1053
+ value=True,
1054
+ info="Most closed models, such as GPT, Claude and Gemini, don't.",
1055
+ )
1056
+ return max_price, max_size, include_undisclosed
1057
+
1058
+
1059
+ def build_demo() -> gr.Blocks:
1060
+ if CATALOG is None:
1061
+ with gr.Blocks(title="ELBench-live") as demo:
1062
+ gr.Markdown(f"ELBench-live data did not load.\n\n```\n{_load_catalog.error}\n```")
1063
+ with gr.Row(visible=False):
1064
+ _noop_btn = gr.Button("noop")
1065
+ _noop_out = gr.Textbox()
1066
+ _noop_btn.click(_zerogpu_noop, [], [_noop_out], api_name="healthz")
1067
+ return demo
1068
+
1069
+ initial_runs = LEADERBOARD_STORE.load_runs()
1070
+ initial_prices, _ = price_snapshot()
1071
+ init_fig, init_rows, init_md = leaderboard_view(
1072
+ "overall", "", "blended_per_1m", True, PRICE_FILTER_MAX, SIZE_FILTER_MAX, True, initial_prices
1073
+ )
1074
+ primary_choices, baseline_choices = build_comparison_choices(initial_runs)
1075
+ default_primary = primary_choices[0][1] if primary_choices else "official:gpt-5.4"
1076
+ default_baselines = [k for k in ("official:deepseek-v4-flash", "official:gpt-5.4") if k != default_primary]
1077
+ granularity_default = "By Source File / Task (Matched Subset)"
1078
+ init_comp_headers, init_comp_rows, init_comp_note = build_comparison_table(
1079
+ default_primary, default_baselines, granularity_default, initial_runs
1080
+ )
1081
+ default_files = CATALOG.default_files()
1082
+
1083
+ with gr.Blocks(title="ELBench-live") as demo:
1084
+ prices = gr.State(initial_prices)
1085
+
1086
+ with gr.Tabs() as tabs:
1087
+ with gr.Tab("Leaderboard", id="leaderboard"):
1088
+ gr.Markdown(
1089
+ "# How well do AI models handle education?\n"
1090
+ "ELBench scores AI models on 2,939 questions in four areas: **teaching**, **safety**, "
1091
+ "**knowledge & reasoning**, and **character & thinking skills**. Each dot is a model: "
1092
+ "higher scores better, further left costs less."
1093
+ )
1094
+ focus = gr.Radio(label="Area", choices=FOCUS_CHOICES, value="overall")
1095
+ chart_max_price, chart_max_size, chart_undisclosed = _filter_controls()
1096
+ chart = gr.Plot(value=init_fig, show_label=False)
1097
+ takeaway = gr.Markdown(value=init_md)
1098
+ with gr.Row():
1099
+ go_test = gr.Button("Test your own model →", variant="primary")
1100
+ refresh = gr.Button("Refresh", variant="secondary")
1101
+
1102
+ with gr.Accordion("More ways to look at the results", open=False):
1103
+ with gr.Row():
1104
+ single_task = gr.Dropdown(
1105
+ label="One task only",
1106
+ choices=TASK_CHOICES,
1107
+ value="",
1108
+ filterable=True,
1109
+ info="Show the scores for a single task instead of a whole area.",
1110
+ )
1111
+ cost_metric = gr.Dropdown(
1112
+ label="Price measure",
1113
+ choices=pareto_cost_metric_choices(),
1114
+ value="blended_per_1m",
1115
+ )
1116
+ include_visitor_runs = gr.Checkbox(label="Show visitor runs", value=True)
1117
+ value_table = gr.Dataframe(
1118
+ headers=["", "Model", "Source", "OpenRouter id", "Input $/1M", "Output $/1M", "Price", "Score"],
1119
+ value=init_rows,
1120
+ wrap=True,
1121
+ show_label=False,
1122
+ )
1123
+
1124
+ with gr.Accordion("Published results (all 2,939 questions)", open=False):
1125
+ gr.Dataframe(
1126
+ headers=[
1127
+ "Rank",
1128
+ "Model",
1129
+ "Overall [95% range]",
1130
+ "Knowledge & reasoning",
1131
+ "Safety",
1132
+ "Teaching",
1133
+ "Character & thinking",
1134
+ "Questions",
1135
+ ],
1136
+ value=official_leaderboard_rows(),
1137
+ wrap=True,
1138
+ show_label=False,
1139
+ )
1140
+ official_model_filter = gr.Dropdown(
1141
+ label="Task-by-task scores for",
1142
+ choices=[("All models", "All Official Models")]
1143
+ + [m["display_name"] for m in load_official_models()],
1144
+ value="GPT-5.4",
1145
+ )
1146
+ official_file_table = gr.Dataframe(
1147
+ headers=["Model", "Area", "Task", "Questions", "Score", "Pass rate (%)", "Mean raw score"],
1148
+ value=official_per_file_rows("GPT-5.4"),
1149
+ wrap=True,
1150
+ show_label=False,
1151
+ )
1152
+
1153
+ with gr.Accordion("Visitor runs", open=False):
1154
+ runs_status = gr.Markdown(f"{len(initial_runs)} visitor run(s). {LEADERBOARD_STORE.sync_status}")
1155
+ runs_table = gr.Dataframe(
1156
+ headers=[
1157
+ "Run id",
1158
+ "Updated (UTC)",
1159
+ "Model",
1160
+ "Judge",
1161
+ "Status",
1162
+ "Scope",
1163
+ "Answers",
1164
+ "Errors",
1165
+ "Overall",
1166
+ "Knowledge & reasoning",
1167
+ "Safety",
1168
+ "Teaching",
1169
+ "Character & thinking",
1170
+ ],
1171
+ value=custom_leaderboard_rows(initial_runs),
1172
+ wrap=True,
1173
+ show_label=False,
1174
+ )
1175
+
1176
+ with gr.Accordion("Compare results side by side", open=False):
1177
+ with gr.Row():
1178
+ comp_primary = gr.Dropdown(
1179
+ label="Result", choices=primary_choices, value=default_primary, filterable=True
1180
+ )
1181
+ comp_baselines = gr.Dropdown(
1182
+ label="Compare with",
1183
+ choices=baseline_choices,
1184
+ value=default_baselines,
1185
+ multiselect=True,
1186
+ filterable=True,
1187
+ )
1188
+ comp_granularity = gr.Radio(
1189
+ label="Detail",
1190
+ choices=[
1191
+ ("Task by task (only tasks the first result covers)", "By Source File / Task (Matched Subset)"),
1192
+ ("By area", "By Module (0–100 Normalized)"),
1193
+ ],
1194
+ value=granularity_default,
1195
+ )
1196
+ comp_note = gr.Markdown(value=init_comp_note)
1197
+ comp_table = gr.Dataframe(
1198
+ headers=init_comp_headers, value=init_comp_rows, wrap=True, show_label=False
1199
+ )
1200
+
1201
+ with gr.Tab("Test a model", id="test"):
1202
+ gr.Markdown(
1203
+ "# Test a model on ELBench\n"
1204
+ "Bring an [OpenRouter API key](" + OPENROUTER_KEYS_PAGE + "), pick a model, and see how it "
1205
+ "does. You'll see the expected cost before anything is charged, and the run stops at your "
1206
+ "spending limit."
1207
+ )
1208
+ api_key = gr.Textbox(
1209
+ label="OpenRouter API key",
1210
+ type="password",
1211
+ placeholder="sk-or-...",
1212
+ info="Used only for this run. Never saved or shown.",
1213
+ )
1214
+ model = gr.Dropdown(
1215
+ label="Model to test",
1216
+ choices=[],
1217
+ value=DEFAULT_TARGET_SLUG,
1218
+ allow_custom_value=True,
1219
+ filterable=True,
1220
+ info="Type to search, or paste any OpenRouter model id.",
1221
+ )
1222
+ with gr.Accordion("Narrow the list by price or size", open=False):
1223
+ pick_max_price, pick_max_size, pick_undisclosed = _filter_controls()
1224
+ models_shown = gr.Markdown()
1225
+ preset = gr.Radio(label="How much to run", choices=_preset_choices(), value=PRESET_QUICK)
1226
+ with gr.Row():
1227
+ total_budget = gr.Number(
1228
+ label="Spending limit (USD)",
1229
+ value=DEFAULT_BUDGET_USD,
1230
+ minimum=0,
1231
+ info="The run pauses here and keeps its progress. Leave empty for no limit.",
1232
+ )
1233
+ persist_to_leaderboard = gr.Checkbox(
1234
+ label="Add my results to the public leaderboard",
1235
+ value=True,
1236
+ info="Shares the model name, settings, answers and scores. Never your key.",
1237
+ )
1238
+
1239
+ with gr.Accordion("Advanced settings", open=False):
1240
+ gr.Markdown("**Judge and simulated student**")
1241
+ with gr.Row():
1242
+ judge_model = gr.Dropdown(
1243
+ label="Judge model",
1244
+ choices=[],
1245
+ value=DEFAULT_CHAT_JUDGE_SLUG,
1246
+ allow_custom_value=True,
1247
+ filterable=True,
1248
+ info="Grades open-ended answers. The default is the closest OpenRouter match to the paper's judge.",
1249
+ )
1250
+ use_system_one_judge = gr.Checkbox(
1251
+ label="Use a decision model as the judge",
1252
+ value=False,
1253
+ info="Cheaper verdict-only models (System 1). Not the paper's setup.",
1254
+ )
1255
+ with gr.Row():
1256
+ use_multiturn_chat = gr.Checkbox(
1257
+ label="Use a separate model as the simulated student",
1258
+ value=False,
1259
+ info="The tutoring task needs a student to talk to. By default the judge model plays the student.",
1260
+ )
1261
+ multiturn_model = gr.Dropdown(
1262
+ label="Simulated student",
1263
+ choices=[],
1264
+ value=DEFAULT_MULTITURN_CHAT_SLUG,
1265
+ visible=False,
1266
+ interactive=False,
1267
+ allow_custom_value=True,
1268
+ filterable=True,
1269
+ )
1270
+ gr.Markdown("**Separate spending limits** (optional, in USD; they apply within the overall limit)")
1271
+ with gr.Row():
1272
+ budget_target = gr.Number(label="Model under test", value=None, minimum=0)
1273
+ budget_judge = gr.Number(label="Judge", value=None, minimum=0)
1274
+ budget_student = gr.Number(label="Simulated student", value=None, minimum=0)
1275
+ gr.Markdown("**Questions**")
1276
+ modules = gr.CheckboxGroup(
1277
+ label="Areas",
1278
+ choices=_module_choices(),
1279
+ value=CATALOG.default_modules(),
1280
+ info="Choosing an area selects all of its tasks.",
1281
+ )
1282
+ files = gr.CheckboxGroup(
1283
+ label="Tasks",
1284
+ choices=_file_choices(list(CATALOG.files)),
1285
+ value=[entry.canonical_name for entry in default_files],
1286
+ )
1287
+ dimensions = gr.CheckboxGroup(
1288
+ label="Only these skills (leave empty for all)",
1289
+ choices=_dimension_choices(default_files),
1290
+ value=[],
1291
+ )
1292
+ with gr.Row():
1293
+ max_samples = gr.Number(
1294
+ label="Stop after this many questions",
1295
+ value=QUICK_CHECK_ITEMS,
1296
+ precision=0,
1297
+ info="Leave empty to run every selected question.",
1298
+ )
1299
+ concurrency = gr.Slider(
1300
+ label="Requests at once",
1301
+ minimum=1,
1302
+ maximum=100,
1303
+ step=1,
1304
+ value=8,
1305
+ info="Higher is faster but more likely to hit rate limits.",
1306
+ )
1307
+ with gr.Row():
1308
+ resume_previous = gr.Checkbox(
1309
+ label="Continue an unfinished run with the same settings", value=True
1310
+ )
1311
+ accept_unpriced = gr.Checkbox(
1312
+ label="Run even though this model has no listed price", value=False
1313
+ )
1314
+
1315
+ estimate_button = gr.Button("Estimate cost", variant="secondary")
1316
+ report = gr.Markdown()
1317
+ with gr.Accordion("Technical details", open=False):
1318
+ report_details = gr.Markdown()
1319
+ confirmed = gr.Checkbox(label="Charge my OpenRouter key for this run, up to my spending limit")
1320
+ run_button = gr.Button("Run", variant="primary")
1321
+ status = gr.Markdown()
1322
+ with gr.Accordion("Detailed results", open=False):
1323
+ table = gr.Dataframe(
1324
+ headers=["Group", "Name", "Count", "Mean score", "Pass rate"], wrap=True, show_label=False
1325
+ )
1326
+ download = gr.File(label="Download answers, scores and errors", file_count="multiple")
1327
+
1328
+ with gr.Tab("About", id="about"):
1329
+ gr.Markdown(_about_markdown())
1330
+
1331
+ # Leaderboard events
1332
+ view_controls = [
1333
+ focus,
1334
+ single_task,
1335
+ cost_metric,
1336
+ include_visitor_runs,
1337
+ chart_max_price,
1338
+ chart_max_size,
1339
+ chart_undisclosed,
1340
+ ]
1341
+ view_outputs = [chart, value_table, takeaway]
1342
+ for control in view_controls:
1343
+ control.change(leaderboard_view, view_controls + [prices], view_outputs)
1344
+ refresh.click(
1345
+ refresh_leaderboards,
1346
+ view_controls + [comp_primary, comp_baselines, comp_granularity, prices],
1347
+ [chart, value_table, takeaway, runs_table, runs_status, comp_primary, comp_baselines, comp_table, comp_note],
1348
+ )
1349
+ go_test.click(lambda: gr.Tabs(selected="test"), None, tabs)
1350
+ official_model_filter.change(official_per_file_rows, [official_model_filter], [official_file_table])
1351
+ for control in (comp_primary, comp_baselines, comp_granularity):
1352
+ control.change(update_comparison, [comp_primary, comp_baselines, comp_granularity], [comp_table, comp_note])
1353
+
1354
+ # Test-a-model events
1355
+ pick_filters = [pick_max_price, pick_max_size, pick_undisclosed]
1356
+ demo.load(
1357
+ load_model_pickers,
1358
+ [use_system_one_judge] + pick_filters,
1359
+ [model, judge_model, multiturn_model, prices, models_shown],
1360
+ )
1361
+ for control in pick_filters:
1362
+ control.change(filter_target_models, [model] + pick_filters, [model, models_shown])
1363
+ use_system_one_judge.change(on_toggle_system_one, [use_system_one_judge], [judge_model])
1364
+ use_multiturn_chat.change(on_toggle_multiturn, [use_multiturn_chat], [multiturn_model])
1365
+ preset.change(apply_preset, [preset], [modules, files, dimensions, max_samples])
1366
+ modules.input(on_modules, [modules], [files, dimensions])
1367
+ files.input(on_files, [files], [dimensions])
1368
+ run_inputs = [
1369
+ api_key,
1370
+ model,
1371
+ judge_model,
1372
+ use_system_one_judge,
1373
+ use_multiturn_chat,
1374
+ multiturn_model,
1375
+ modules,
1376
+ files,
1377
+ dimensions,
1378
+ max_samples,
1379
+ concurrency,
1380
+ prices,
1381
+ total_budget,
1382
+ budget_target,
1383
+ budget_judge,
1384
+ budget_student,
1385
+ ]
1386
+ estimate_button.click(estimate, run_inputs, [report, report_details])
1387
+ run_button.click(
1388
+ run_evaluation,
1389
+ run_inputs + [confirmed, accept_unpriced, resume_previous, persist_to_leaderboard],
1390
+ [status, table, download],
1391
+ )
1392
+ with gr.Row(visible=False):
1393
+ _noop_btn = gr.Button("noop")
1394
+ _noop_out = gr.Textbox()
1395
+ _noop_btn.click(_zerogpu_noop, [], [_noop_out], api_name="healthz")
1396
+ return demo
1397
+
1398
+
1399
+ demo = build_demo()
1400
+ # Spaces imports this module and calls launch() itself, so the queue has to be set here.
1401
+ demo.queue(default_concurrency_limit=1)
1402
+
1403
+
1404
+ if __name__ == "__main__":
1405
+ demo.launch(theme=gr.themes.Soft())
docs/ARCHITECTURE.md ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench Architecture
2
+
3
+ This document describes the engineering design of ELBench: the registry, the
4
+ provider abstraction, the execution model, and how to extend the framework.
5
+ For an overview of the benchmark and the leaderboard, see the top-level
6
+ [README](../README.md).
7
+
8
+ ELBench is built as an engineering system, not a one-off script. Four benchmark
9
+ modules are kept in one consistent framework:
10
+
11
+ - `安全可信` / Safety & Trustworthiness
12
+ - `高阶育人` / High-Level Educational Cultivation
13
+ - `基本教育` / Basic Education
14
+ - `通用模型` / General Capability
15
+
16
+ ## Principles
17
+
18
+ - Registry-driven loading instead of hardcoded file logic
19
+ - Provider adapters instead of provider-specific branches in the runner
20
+ - Rule-based judging for objective tasks
21
+ - `LLM-as-a-Judge` for subjective and open-ended tasks
22
+ - Stable machine-readable outputs, logs, checkpoints, and summaries
23
+ - Resume support after interruption
24
+ - Configurable concurrency with a default global cap of `100`
25
+
26
+ ## What Is Actually Wired Today
27
+
28
+ ### Registered benchmark files
29
+
30
+ `configs/file_registry.yaml` currently resolves these files:
31
+
32
+ - `安全拒答.jsonl`
33
+ - `安全引导.jsonl`
34
+ - `安全回答.jsonl`
35
+ - `SATAs.xlsx`
36
+ - `adversarial_prompts.xlsx`
37
+ - `高阶育人-edu.jsonl`
38
+ - `高阶育人-omni.jsonl`
39
+ - `knowledge.yaml`
40
+ - `question.yaml`
41
+ - `cross.yaml`
42
+ - `config_guided_task.yaml`
43
+ - `mmlu_pro_sampled.jsonl`
44
+ - `ceval_sampled.jsonl`
45
+ - `ifeval_sampled.jsonl`
46
+ - `math_500_sampled.jsonl`
47
+ - `aime24_sampled.jsonl`
48
+ - `aime25.jsonl`
49
+ - `aime26.jsonl`
50
+ - `gsm8k_sampled.jsonl`
51
+
52
+ Files are only considered active benchmark inputs after they are registered.
53
+ Data files not present in `configs/file_registry.yaml` are not part of the
54
+ runnable benchmark.
55
+
56
+ ### Current judge strategy
57
+
58
+ Objective tasks use rule or reference matching:
59
+
60
+ - `SATAs.xlsx`
61
+ - `高阶育人-omni.jsonl`
62
+ - `mmlu_pro_sampled.jsonl`
63
+ - `ceval_sampled.jsonl`
64
+ - `ifeval_sampled.jsonl`
65
+ - `math_500_sampled.jsonl`
66
+ - `aime24_sampled.jsonl`
67
+ - `aime25.jsonl`
68
+ - `aime26.jsonl`
69
+ - `gsm8k_sampled.jsonl`
70
+
71
+ Subjective or open-ended tasks use `LLM-as-a-Judge`:
72
+
73
+ - `安全拒答.jsonl`
74
+ - `安全引导.jsonl`
75
+ - `安全回答.jsonl`
76
+ - `adversarial_prompts.xlsx`
77
+ - `高阶育人-edu.jsonl`
78
+
79
+ The repository still treats external provider integration as pluggable.
80
+ `mock.default` and `mock.judge` are included so the full pipeline can be
81
+ validated locally without real API access.
82
+
83
+ ## Repository Layout
84
+
85
+ ```text
86
+ .
87
+ ├── configs/
88
+ │ ├── app.yaml
89
+ │ ├── basic_education.yaml
90
+ │ ├── field_mappings.yaml
91
+ │ ├── file_registry.yaml
92
+ │ ├── judges.yaml
93
+ │ ├── models.yaml
94
+ │ ├── module_registry.yaml
95
+ │ └── providers.yaml
96
+ ├── data/
97
+ │ └── benchmark_root/
98
+ │ ├── 安全可信/
99
+ │ ├── 高阶育人/
100
+ │ ├── 基本教育/
101
+ │ └── 通用/
102
+ ├── docs/
103
+ │ └── TEAM_MAINTENANCE_GUIDE.zh-CN.md
104
+ ├── results/
105
+ ├── scripts/
106
+ │ └── run_benchmark.py
107
+ ├── src/
108
+ │ └── elbench/
109
+ │ ├── basic_education_runtime/
110
+ │ ├── cli.py
111
+ │ ├── config/
112
+ │ ├── execution/
113
+ │ ├── judges/
114
+ │ ├── loaders/
115
+ │ ├── persistence/
116
+ │ ├── providers/
117
+ │ ├── registry/
118
+ │ ├── schemas/
119
+ │ ├── summary/
120
+ │ └── utils/
121
+ └── tests/
122
+ ```
123
+
124
+ `scripts/run_benchmark.py` is intentionally a thin entrypoint. The actual
125
+ benchmark logic lives under `src/elbench/`.
126
+
127
+ ## Core Architecture
128
+
129
+ ### Registry and schema adaptation
130
+
131
+ - `configs/file_registry.yaml` maps files to module, subset, task, loader, and
132
+ canonical name.
133
+ - `configs/field_mappings.yaml` maps raw dataset fields into ELBench's unified
134
+ internal schema.
135
+ - JSONL, XLSX, and basic-education YAML inputs are normalized into the same
136
+ `Sample` structure.
137
+
138
+ ### Unified sample/result schema
139
+
140
+ Loaded records are normalized into a stable schema with fields such as:
141
+
142
+ - `sample_id`
143
+ - `source_file`
144
+ - `module`
145
+ - `subset`
146
+ - `task`
147
+ - `dimension`
148
+ - `prompt`
149
+ - `reference`
150
+ - `metadata`
151
+
152
+ Execution output includes:
153
+
154
+ - `provider_name`
155
+ - `model_id`
156
+ - `model_name`
157
+ - `model_response`
158
+ - `judge_result`
159
+ - `score`
160
+ - `judge_reason`
161
+ - `judge_metadata`
162
+ - `latency_ms`
163
+ - `retry_count`
164
+ - `timestamp`
165
+
166
+ ### Provider abstraction
167
+
168
+ - `ModelClient` is the common interface for model execution.
169
+ - Provider-specific API differences are isolated under `src/elbench/providers/`.
170
+ - The main runner does not care whether a model comes from OpenAI, Anthropic,
171
+ Gemini, or another vendor once the adapter exists.
172
+
173
+ ### Execution, concurrency, and recovery
174
+
175
+ The runner supports:
176
+
177
+ - global concurrency control
178
+ - provider/model-level concurrency limits
179
+ - retry with exponential backoff
180
+ - failure logging
181
+ - checkpoint-based resume
182
+ - raw-response persistence
183
+ - judged-result persistence
184
+ - summary generation
185
+
186
+ Default global max concurrency is `100` in `configs/app.yaml`.
187
+
188
+ ### Basic Education runtime
189
+
190
+ `基本教育` differs from the other modules only in execution style, not in data
191
+ governance:
192
+
193
+ - benchmark data stays under `data/benchmark_root/基本教育/`
194
+ - file discovery still goes through `configs/file_registry.yaml`
195
+ - runtime execution is handled by `src/elbench/execution/basic_education.py`
196
+ - the internal runtime implementation lives under
197
+ `src/elbench/basic_education_runtime/`
198
+
199
+ This internal runtime was adapted from the ELMES project, but it is now packaged
200
+ and maintained as part of ELBench itself.
201
+
202
+ ## Output Layout
203
+
204
+ ```text
205
+ outputs/
206
+ ├── raw_responses/
207
+ ├── judged_results/
208
+ ├── summaries/
209
+ └── logs/
210
+ ```
211
+
212
+ These directories are runtime artifacts and are intentionally ignored by Git.
213
+ Curated leaderboards and audit summaries are committed under `results/`.
214
+
215
+ ## Configuration Surface
216
+
217
+ ### `configs/models.yaml`
218
+
219
+ Concrete evaluated-model and judge-model instances, including timeout, retry,
220
+ rate limits, and provider kwargs.
221
+
222
+ ### `configs/providers.yaml`
223
+
224
+ Provider adapter registry plus default provider capabilities. Current built-in
225
+ adapters:
226
+
227
+ - `mock`
228
+ - `openai_compatible`
229
+
230
+ ### `configs/judges.yaml`
231
+
232
+ Task-to-judge routing:
233
+
234
+ - `rule`
235
+ - `llm`
236
+
237
+ ### `configs/basic_education.yaml`
238
+
239
+ Scenario-level runtime settings for the internal basic education runtime. This
240
+ file does not replace the registry; it complements it.
241
+
242
+ ## Extending ELBench
243
+
244
+ ### Add a new provider
245
+
246
+ 1. Implement an adapter under `src/elbench/providers/`.
247
+ 2. Register it in `configs/providers.yaml`.
248
+ 3. Add concrete models in `configs/models.yaml`.
249
+
250
+ ### Add a new benchmark file
251
+
252
+ 1. Put the data file under `data/benchmark_root/`.
253
+ 2. Register it in `configs/file_registry.yaml`.
254
+ 3. Add field mappings in `configs/field_mappings.yaml`.
255
+ 4. Route judging in `configs/judges.yaml`.
256
+ 5. Add or reuse tests.
257
+
258
+ ### Add a future module
259
+
260
+ ELBench already reserves space for further modules. The intended path remains:
261
+
262
+ 1. add module registration
263
+ 2. add file registration
264
+ 3. add field mappings
265
+ 4. add judge routing
266
+ 5. add task-specific logic only if configuration cannot express the behavior
267
+
268
+ No main-runner rewrite should be required.
269
+
270
+ ## Maintenance
271
+
272
+ The canonical team maintenance guide is
273
+ [`docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md`](TEAM_MAINTENANCE_GUIDE.zh-CN.md). That
274
+ document explains what each maintained directory is for, which paths are
275
+ runtime-only, and what should or should not be committed.
276
+
277
+ ## Status
278
+
279
+ Implemented:
280
+
281
+ - config system
282
+ - file registry
283
+ - JSONL/XLSX/YAML loaders
284
+ - unified sample/result schema
285
+ - provider abstraction
286
+ - concurrent runner
287
+ - retries and checkpoint resume
288
+ - raw/judged/summary/log outputs
289
+ - rule judges for objective tasks
290
+ - `LLM-as-a-Judge` execution path for subjective tasks
291
+ - built-in basic-education bridge on top of the internal runtime module
292
+
293
+ Still intentionally unfinished:
294
+
295
+ - production-grade adapters for every external vendor
296
+ - finalized subjective judge rubrics and prompts
297
+ - complete future-module integration beyond the current registered files
docs/ARCHITECTURE.zh-CN.md ADDED
@@ -0,0 +1,274 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench 架构说明
2
+
3
+ 本文档描述 ELBench 的工程设计:注册表、provider 抽象、执行模型,以及如何扩展框架。基准与排行榜的总览见顶层 [README](../README.zh-CN.md)。
4
+
5
+ ELBench 不是临时脚本,而是按正式工程维护的评测系统。当前仓库在一个统一框架下承载四个模块:
6
+
7
+ - `安全可信` / Safety & Trustworthiness
8
+ - `高阶育人` / High-Level Educational Cultivation
9
+ - `基本教育` / Basic Education
10
+ - `通用模型` / General Capability
11
+
12
+ `基本教育` 已作为 ELBench 的正式模块接入。它的 benchmark 数据放在 `data/benchmark_root/基本教育/`,多轮执行由项目内部的 `basic_education_runtime` 模块负责,代码位于 `src/elbench/basic_education_runtime/`。这个内部运行时来源于 ELMES 项目,但现在已经按 ELBench 的内部模块方式接管和维护。
13
+
14
+ ## 设计原则
15
+
16
+ - 评测流程由注册表和配置驱动,不在主流程里写死文件逻辑
17
+ - 模型调用统一走 provider adapter,不让厂商差异污染主 runner
18
+ - 客观题优先规则判分
19
+ - 主观和开放式任务采用 `LLM-as-a-Judge`
20
+ - 稳定的机器可读输出、日志、检查点与汇总
21
+ - 中断后支持续跑
22
+ - 可配置并发,默认全局上限 `100`
23
+
24
+ ## 当前实际接入的内容
25
+
26
+ ### 已注册的 benchmark 文件
27
+
28
+ `configs/file_registry.yaml` 当前解析以下文件:
29
+
30
+ - `安全拒答.jsonl`
31
+ - `安全引导.jsonl`
32
+ - `安全回答.jsonl`
33
+ - `SATAs.xlsx`
34
+ - `adversarial_prompts.xlsx`
35
+ - `高阶育人-edu.jsonl`
36
+ - `高阶育人-omni.jsonl`
37
+ - `knowledge.yaml`
38
+ - `question.yaml`
39
+ - `cross.yaml`
40
+ - `config_guided_task.yaml`
41
+ - `mmlu_pro_sampled.jsonl`
42
+ - `ceval_sampled.jsonl`
43
+ - `ifeval_sampled.jsonl`
44
+ - `math_500_sampled.jsonl`
45
+ - `aime24_sampled.jsonl`
46
+ - `aime25.jsonl`
47
+ - `aime26.jsonl`
48
+ - `gsm8k_sampled.jsonl`
49
+
50
+ 文件只有在注册之后才会被视为有效的 benchmark 输入。没有出现在 `configs/file_registry.yaml` 中的数据文件,不属于可运行基准的一部分。
51
+
52
+ ### 当前判分策略
53
+
54
+ 客观题用规则或参考答案匹配:
55
+
56
+ - `SATAs.xlsx`
57
+ - `高阶育人-omni.jsonl`
58
+ - `mmlu_pro_sampled.jsonl`
59
+ - `ceval_sampled.jsonl`
60
+ - `ifeval_sampled.jsonl`
61
+ - `math_500_sampled.jsonl`
62
+ - `aime24_sampled.jsonl`
63
+ - `aime25.jsonl`
64
+ - `aime26.jsonl`
65
+ - `gsm8k_sampled.jsonl`
66
+
67
+ 主观或开放式任务用 `LLM-as-a-Judge`:
68
+
69
+ - `安全拒答.jsonl`
70
+ - `安全引导.jsonl`
71
+ - `安全回答.jsonl`
72
+ - `adversarial_prompts.xlsx`
73
+ - `高阶育人-edu.jsonl`
74
+
75
+ 仓库把外部 provider 接入设计为可插拔。内置 `mock.default` 和 `mock.judge`,可在没有真实 API 的情况下本地验证完整流水线。
76
+
77
+ ## 仓库结构
78
+
79
+ ```text
80
+ .
81
+ ├── configs/
82
+ │ ├── app.yaml
83
+ │ ├── basic_education.yaml
84
+ │ ├── field_mappings.yaml
85
+ │ ├── file_registry.yaml
86
+ │ ├── judges.yaml
87
+ │ ├── models.yaml
88
+ │ ├── module_registry.yaml
89
+ │ └── providers.yaml
90
+ ├── data/
91
+ │ └── benchmark_root/
92
+ │ ├── 安全可信/
93
+ │ ├── 高阶育人/
94
+ │ ├── 基本教育/
95
+ │ └── 通用/
96
+ ├── docs/
97
+ │ └── TEAM_MAINTENANCE_GUIDE.zh-CN.md
98
+ ├── results/
99
+ ├── scripts/
100
+ │ └── run_benchmark.py
101
+ ├── src/
102
+ │ └── elbench/
103
+ │ ├── basic_education_runtime/
104
+ │ ├── cli.py
105
+ │ ├── config/
106
+ │ ├── execution/
107
+ │ ├── judges/
108
+ │ ├── loaders/
109
+ │ ├── persistence/
110
+ │ ├── providers/
111
+ │ ├── registry/
112
+ │ ├── schemas/
113
+ │ ├── summary/
114
+ │ └── utils/
115
+ └── tests/
116
+ ```
117
+
118
+ `scripts/run_benchmark.py` 刻意只是一个薄入口,真正的评测逻辑都在 `src/elbench/` 下。
119
+
120
+ ## 核心架构
121
+
122
+ ### 注册表与 schema 适配
123
+
124
+ - `configs/file_registry.yaml` 把文件映射到模块、子集、任务、loader 与规范名。
125
+ - `configs/field_mappings.yaml` 把原始数据集字段映射进 ELBench 的统一内部 schema。
126
+ - JSONL、XLSX、基本教育 YAML 输入都被规范化为同一个 `Sample` 结构。
127
+
128
+ ### 统一的 sample/result schema
129
+
130
+ 加载后的记录被规范化为稳定 schema,字段包括:
131
+
132
+ - `sample_id`
133
+ - `source_file`
134
+ - `module`
135
+ - `subset`
136
+ - `task`
137
+ - `dimension`
138
+ - `prompt`
139
+ - `reference`
140
+ - `metadata`
141
+
142
+ 执行输出包括:
143
+
144
+ - `provider_name`
145
+ - `model_id`
146
+ - `model_name`
147
+ - `model_response`
148
+ - `judge_result`
149
+ - `score`
150
+ - `judge_reason`
151
+ - `judge_metadata`
152
+ - `latency_ms`
153
+ - `retry_count`
154
+ - `timestamp`
155
+
156
+ ### Provider 抽象
157
+
158
+ - `ModelClient` 是模型执行的统一接口。
159
+ - 厂商 API 差异都隔离在 `src/elbench/providers/` 下。
160
+ - 一旦 adapter 存在,主 runner 不关心模型来自 OpenAI、Anthropic、Gemini 还是其他厂商。
161
+
162
+ ### 执行、并发与恢复
163
+
164
+ runner 支持:
165
+
166
+ - 全局并发控制
167
+ - provider/模型级并发上限
168
+ - 指数退避重试
169
+ - 失败日志
170
+ - 基于检查点的续跑
171
+ - 原始响应持久化
172
+ - 判分结果持久化
173
+ - 汇总生成
174
+
175
+ `configs/app.yaml` 中默认全局最大并发为 `100`。
176
+
177
+ ### 基本教育运行时
178
+
179
+ `基本教育` 与其他模块的差别只在执行方式,不在数据治理:
180
+
181
+ - benchmark 数据仍放在 `data/benchmark_root/基本教育/`
182
+ - 文件发现仍走 `configs/file_registry.yaml`
183
+ - 运行时执行由 `src/elbench/execution/basic_education.py` 负责
184
+ - 内部运行时实现位于 `src/elbench/basic_education_runtime/`
185
+
186
+ ## 输出结构
187
+
188
+ ```text
189
+ outputs/
190
+ ├── raw_responses/
191
+ ├── judged_results/
192
+ ├── summaries/
193
+ └── logs/
194
+ ```
195
+
196
+ 这些目录是运行时产物,刻意被 Git 忽略。整理后的排行榜与审计汇总提交在 `results/` 下。
197
+
198
+ ## 配置面
199
+
200
+ ### `configs/models.yaml`
201
+
202
+ 具体的被测模型与裁判模型实例,含超时、重试、限速与 provider 参数。
203
+
204
+ ### `configs/providers.yaml`
205
+
206
+ provider adapter 注册表与默认 provider 能力。当前内置 adapter:
207
+
208
+ - `mock`
209
+ - `openai_compatible`
210
+
211
+ ### `configs/judges.yaml`
212
+
213
+ 任务到裁判的路由:
214
+
215
+ - `rule`
216
+ - `llm`
217
+
218
+ ### `configs/basic_education.yaml`
219
+
220
+ 基本教育内部运行时的场景级设置。该文件不替代注册表,而是对其补充。
221
+
222
+ ## 扩展 ELBench
223
+
224
+ ### 新增 provider
225
+
226
+ 1. 在 `src/elbench/providers/` 下实现 adapter。
227
+ 2. 在 `configs/providers.yaml` 中注册。
228
+ 3. 在 `configs/models.yaml` 中加入具体模型。
229
+
230
+ ### 新增 benchmark 文件
231
+
232
+ 1. 把数据文件放到 `data/benchmark_root/` 下。
233
+ 2. 在 `configs/file_registry.yaml` 中注册。
234
+ 3. 在 `configs/field_mappings.yaml` 中加字段映射。
235
+ 4. 在 `configs/judges.yaml` 中路由判分。
236
+ 5. 新增或复用测试。
237
+
238
+ ### 新增未来模块
239
+
240
+ ELBench 已为更多模块预留空间,预期路径仍是:
241
+
242
+ 1. 加模块注册
243
+ 2. 加文件注册
244
+ 3. 加字段映射
245
+ 4. 加裁判路由
246
+ 5. 仅当配置无法表达行为时才加任务专属逻辑
247
+
248
+ 不需要重写主 runner。
249
+
250
+ ## 维护
251
+
252
+ 权威的团队维护指南是 [`docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md`](TEAM_MAINTENANCE_GUIDE.zh-CN.md),它说明每个维护目录的用途、哪些路径仅运行时、以及哪些该提交哪些不该提交。
253
+
254
+ ## 状态
255
+
256
+ 已实现:
257
+
258
+ - 配置系统
259
+ - 文件注册表
260
+ - JSONL/XLSX/YAML loader
261
+ - 统一 sample/result schema
262
+ - provider 抽象
263
+ - 并发 runner
264
+ - 重试与检查点续跑
265
+ - 原始/判分/汇总/日志输出
266
+ - 客观题规则裁判
267
+ - 主观题 `LLM-as-a-Judge` 执行路径
268
+ - 基于内部运行时模块的基本教育桥接
269
+
270
+ 仍刻意未完成:
271
+
272
+ - 面向每个外部厂商的生产级 adapter
273
+ - 最终化的主观裁判 rubric 与 prompt
274
+ - 超出当前已注册文件的完整未来模块接入
docs/DASHBOARD_TARGETS.md ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench-live dashboard targets
2
+
3
+ Working list for the hosted dashboard (`dashboard/app.py`, `src/elbench/dashboard/`).
4
+ Each target records the issue, the evidence, the agreed resolution, and how we
5
+ know it is done. Work them in order; update **Status** as each one lands.
6
+
7
+ ## Goal
8
+
9
+ 1. Present the published ELBench results clearly.
10
+ 2. Say plainly what ELBench measures.
11
+ 3. Make it plug-and-play for anyone with an OpenRouter key to test a model and
12
+ join the leaderboard by default, with fine-grained control available:
13
+ item selection, judge and student models, and a budget overall and per
14
+ component model.
15
+
16
+ ## Decisions
17
+
18
+ - **D1 Scoring follows the paper.** Module and overall scores are computed the
19
+ way the published README/paper table is, unless there is a documented reason
20
+ to override it. Safety uses the item-weighted mean score, not pass rate
21
+ (`leaderboard_FINAL.json` in the results dataset uses pass rate; the paper
22
+ table does not).
23
+ - **D2 Chart first.** The leaderboard chart is the landing view. Explanation
24
+ lives in an About tab. Advanced controls are collapsed by default.
25
+ - **D3 Participate by default.** Saving a run to the public leaderboard is on
26
+ by default, with an opt-out. Only runs that cover the scored slice in full
27
+ are plotted against the paper models.
28
+
29
+ ## Phase 1 · Correctness
30
+
31
+ ### T1 Refresh stale official per-file data
32
+ - **Issue:** `official_results_data.json` per-file stats predate the results
33
+ dataset's audit fixes, so recomputing module scores from them does not
34
+ reproduce the published table.
35
+ - **Evidence:** High-Level education file for Claude-Opus-4.8, DeepSeek-V4-Pro,
36
+ DeepSeek-V4-Flash, Doubao-Seed-2.0-Pro, GLM-5.1 holds the pre-fix pass rate
37
+ (e.g. DeepSeek-V4-Flash 63.2; post-fix 92.6 per
38
+ `audit-judge-integrity/highedu_llmjudge_fix_summary.json`). High-Level gaps up
39
+ to 15 points. Basic Education per-file data is stale for GPT-5.4 and
40
+ DeepSeek-V4-Pro (`question.yaml` avg 11.36 on a 5-point scale).
41
+ - **Resolution:** Rebuild per-file stats from the audited files in
42
+ `ZeroLoss-Lab/ELBench-results`; record the source files and date in the JSON.
43
+ - **Done when:** a test recomputes every module and overall score from
44
+ per-file data and matches the published table within 0.1 for all 9 models.
45
+ - **Status:** done 2026-10-26. Safety, Basic Education, High-Level and Overall reproduce
46
+ the paper within 0.1; General within 0.25, explained by GSM8K. Provenance is in the JSON's
47
+ `provenance` block; `module_ci` and `overall_ci` added from `bootstrap_ci.json`.
48
+ Test: `test_official_per_file_data_reproduces_the_published_table`.
49
+
50
+ ### T2 One scoring function for paper models and visitor runs
51
+ - **Issue:** `compute_normalized_scores` aggregates differently from the paper.
52
+ - **Evidence:** Safety uses pass rate (paper: mean score; gap up to 2.7 pts).
53
+ Basic Education is item-weighted (paper: simple mean of the 4 scenarios).
54
+ - **Resolution:** General and High-Level: item-weighted pass rate. Safety:
55
+ item-weighted mean score. Basic Education: mean of the 4 scenario scores,
56
+ each normalized to its rubric maximum. Overall: mean of the 4 modules.
57
+ - **Done when:** the T1 test uses this same function.
58
+ - **Status:** done 2026-10-26. `normalize_file_entry` + `module_scores_from_per_file`.
59
+ Visitor runs also skip missing or out-of-range Basic Education scores
60
+ (`basic_education_valid_means`), the rule that reproduces the audited values.
61
+
62
+ ### T3 Keep partial runs off the frontier
63
+ - **Issue:** a run's score covers only the items it ran, but is plotted against
64
+ full-benchmark scores.
65
+ - **Evidence:** a 5-item MMLU run is placed on the Overall frontier at 100%.
66
+ - **Resolution:** a run is plotted for a metric only if it ran every item in
67
+ that metric's scope (all files of the module, or the full benchmark for
68
+ Overall; no `max_samples` or dimension filter). Partial runs stay visible in
69
+ the runs table with their scope.
70
+ - **Done when:** test: the 5-item run is excluded from Overall and General and
71
+ appears on the per-file MMLU view only if that file ran in full.
72
+ - **Status:** done 2026-10-26. `run_covers_metric` compares judged counts per file with the
73
+ published counts (which equal the catalog's 2,939). The chart summary says how many runs were
74
+ left out. Runs now also record their dimension filter.
75
+
76
+ ### T4 Remove the fixed-footprint "Estimated full benchmark cost"
77
+ - **Issue:** the metric applies 0.73M input / 1.5M output tokens to every
78
+ model and leaves out judge cost; it is a price blend labelled as an estimate.
79
+ - **Evidence:** GPT-5.4: metric $24.3; the app's own estimator $3.49–$285.
80
+ - **Resolution:** drop it. Default cost axis is the 3:1 blended price per 1M
81
+ tokens; input and output prices remain as options. Once real full runs
82
+ record measured cost (T12), offer "measured cost of a full run".
83
+ - **Status:** done 2026-10-26 (measured-cost option waits on T12).
84
+
85
+ ### T5 One dated price snapshot
86
+ - **Issue:** first render uses prices stored in the JSON; Refresh uses live
87
+ prices, so points move (DeepSeek-V4-Pro $0.96 → $0.73).
88
+ - **Resolution:** fetch the live catalog once per process (cached, with
89
+ timestamp), fall back to stored prices, and label the chart "Prices from
90
+ OpenRouter, <date>".
91
+ - **Status:** done 2026-10-26. `price_snapshot()` in `dashboard/app.py`, 6-hour cache.
92
+
93
+ ### T6 Show uncertainty
94
+ - **Issue:** the top six are within 2 points with overlapping 95% intervals,
95
+ but the chart presents small differences as wins.
96
+ - **Resolution:** error bars from the published intervals; a plain note when
97
+ the leaders are statistically tied.
98
+ - **Status:** done 2026-10-26 for the current chart: error bars and a "statistically tied" line.
99
+ Carry both into the T13 chart.
100
+
101
+ ### T7 Hygiene
102
+ - `README.md` declares `sdk_version: 6.30.0`; 6.4.0 is installed and
103
+ `dashboard/README.md` says 6.4.0. Pick one.
104
+ - `.env` holds secrets and is not git-ignored.
105
+ - **Status:** done 2026-10-26. README set to 6.4.0 (the version pinned in `pyproject.toml` and
106
+ installed); `.env` added to `.gitignore`.
107
+
108
+ ## Phase 2 · Presentation and participation
109
+
110
+ ### T8 Three tabs: Leaderboard · Test a model · About
111
+ - Leaderboard: one-line description, chart, module focus switch, short
112
+ takeaway, call to action to test a model; tables collapsed below.
113
+ - About: what ELBench measures (four modules in plain words), how scoring
114
+ works, links, and the fidelity notes now in the header.
115
+ - **Status:** done 2026-10-26. `gr.Tabs` with ids; "Test your own model →" switches tabs. About is built from
116
+ `AREA_LABELS` and the catalog counts.
117
+
118
+ ### T9 Plain language
119
+ - Replace jargon in visible labels ("Endpoint to test", "Judge endpoint",
120
+ "/api/v1/systemone", "preflight", "build_summary", "Matched Subset",
121
+ "Pareto-optimal") and drop meta commentary from the main flow. Technical
122
+ detail stays available in the estimate's details and the About tab.
123
+ - **Status:** done 2026-10-26. Visible labels use area names and plain task names (`area_label`, `plain_file`);
124
+ the technical plan moved to a collapsed "Technical details" accordion.
125
+
126
+ ### T10 Plug-and-play run with sensible defaults
127
+ - Key → model → preset (Quick check / Standard / Full) → estimate → run.
128
+ Defaults: official judge, save to leaderboard on. One confirmation.
129
+ - **Status:** done 2026-10-26. Presets Quick check (5 questions, default) / Automatically checked set /
130
+ Full benchmark; $5 default limit; leaderboard opt-in on; key checked with `GET /api/v1/key` (shows credit
131
+ left); model list loaded on page load from the cached OpenRouter snapshot.
132
+
133
+ ### T11 Advanced controls, collapsed
134
+ - Judge model and System 1 judging, simulated-student model, files and
135
+ dimensions, item cap, concurrency, resume, leaderboard opt-out.
136
+ - **Status:** done 2026-10-26. "Advanced settings" accordion holds judge / decision-model judging, simulated
137
+ student, per-component limits, areas/tasks/skills, question cap, concurrency, resume, unpriced models.
138
+
139
+ ### T12 Budgets: overall and per component model
140
+ - **Components:** model under test, judge, simulated student (multi-turn).
141
+ - **Before the run:** show the estimate per component against its budget;
142
+ block the run when an estimate's low end already exceeds a budget.
143
+ - **During the run:** meter spend per component from the `usage.cost` that
144
+ OpenRouter returns; when a component or the total reaches its cap, stop
145
+ through the runner's existing quota-exhausted path, so the run is saved as
146
+ partial and can be resumed. Overshoot is bounded by requests already in flight.
147
+ - **Done when:** unit test stops a run at a cap and leaves it resumable; a
148
+ live check with a tiny cap stops after the first requests.
149
+ - **Status:** done 2026-10-26. `elbench.utils.spend` ledger shared with the runtime subprocess; checks in
150
+ `OpenAICompatibleClient`, `SystemOneJudge`, and a LangChain `SpendCallback`. The estimate blocks a run when a
151
+ low estimate exceeds a limit. Unit tests: `tests/test_spend_limits.py`, `tests/test_spend_runtime_callback.py`.
152
+ Live check (GPT-6 Luna, GLM 5.3 Flash judge, $0.003 total): total limit paused at 3/5 and resumed to 5/5;
153
+ judge limit paused at 2/3 with spend split by role; key absent from every output file. Overshoot seen:
154
+ limit $0.0004, spent $0.0011 (requests in flight finish); the estimate now says so.
155
+
156
+ ## Phase 3 · Filters and chart
157
+
158
+ ### T13 Interactive chart
159
+ - Plotly via `gr.Plot`: hover details, log price axis, error bars, legend
160
+ for paper models vs visitor runs.
161
+ - **Status:** done 2026-10-26. `_render_pareto_figure` returns a Plotly figure: hover shows score, 95% range,
162
+ price and verdict; log price axis with plain-dollar ticks; light error bars; published = blue circles,
163
+ visitor runs = orange diamonds; best-value step line. Labels are placed to avoid dots, whiskers, the line
164
+ and each other, and are dropped (hover still names the point) when no spot is free. Fixed a label bug:
165
+ chart title and axis showed metric keys (`overall`, `blended_per_1m`) instead of their names.
166
+ `plotly` added to `requirements.txt` and the `dashboard` extra.
167
+
168
+ ### T14 Price and size filters
169
+ - OpenRouter `/api/v1/models` has prices but no parameter count. 179 of 468
170
+ models carry `hugging_face_id`; the Hugging Face API returns
171
+ `safetensors.total` (checked: Qwen3-235B → 235.09B). Closed models have no
172
+ size, so the size filter gets an "include undisclosed" toggle. Filter on
173
+ total size and show active size when the model name gives it (`-A22B`).
174
+ Cache sizes; do not fetch on every page load.
175
+ - Apply the filters to both the chart and the model picker.
176
+ - **Status:** done 2026-10-26. `elbench.dashboard.model_sizes`: sizes from Hugging Face `safetensors.total`
177
+ cached in `model_sizes.json` (152 repos; refresh with `python -m elbench.dashboard.model_sizes`), with the
178
+ size in the model id as fallback and for active size. 176 of 468 OpenRouter models have a known size.
179
+ The same three controls (highest price, largest size, include undisclosed) sit above the chart and in the
180
+ model picker. Filtered models leave the chart, table and best-value line, with a note saying how many.
181
+
182
+ ## Testing
183
+
184
+ - Unit tests in `tests/test_dashboard_faithfulness.py` per target.
185
+ - Light live checks against OpenRouter with a key from `.env` (variable
186
+ `OPENROUTER_API_KEY`): model list, one cheap request with `usage.cost`, a
187
+ budget stop. Keep each check under a few cents.
docs/RESULT_STORAGE_SPEC.zh-CN.md ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench 结果保存规范
2
+
3
+ ## 1. 目标
4
+
5
+ 这份规范只解决一件事:让评测结果在仓库里保存得清楚、可追溯、适合提交。
6
+
7
+ - `outputs/` 只放运行时全量产物。
8
+ - `results/` 只放审核后的正式结果包。
9
+ - Git 默认只跟踪 `results/`,不跟踪 `outputs/`。
10
+
11
+ ## 2. 目录边界
12
+
13
+ | 路径 | 是否入库 | 用途 | 说明 |
14
+ |---|---|---|---|
15
+ | `outputs/` | 否 | 运行时全量产物 | 大、杂、可删除,默认由 `.gitignore` 忽略 |
16
+ | `results/` | 是 | 正式结果包 | 小、稳定、适合汇报和复查 |
17
+ | `docs/RESULT_STORAGE_SPEC.zh-CN.md` | 是 | 本规范 | 结果保存规则的唯一规范入口 |
18
+
19
+ `outputs/` 当前标准子目录如下:
20
+
21
+ - `outputs/raw_responses/<run_id>/`
22
+ - `outputs/judged_results/<run_id>/`
23
+ - `outputs/summaries/<run_id>/`
24
+ - `outputs/logs/<run_id>/`
25
+
26
+ ## 3. 标准流程
27
+
28
+ 1. 所有评测先按真实 `run_id` 落到 `outputs/`。
29
+ 2. 跑完后先检查 `summary`、`failures`、日志和样本范围,确认这是不是一次正式可引用的 run。
30
+ 3. 只有正式结果才允许“升格”到 `results/`。
31
+ 4. 升格时只整理最小必要文件,不复制原始响应、checkpoint、重试记录、全量日志。
32
+ 5. 在 `results/` 下创建结果包目录,并补齐 `README.md`、`manifest.json`、`summary.json`。
33
+ 6. 提交 Git 时只提交 `results/` 下的正式结果包和必要文档。
34
+
35
+ ## 4. 命名规范
36
+
37
+ 正式结果包目录名统一为:
38
+
39
+ `results/<model_id>-<scope>-<YYYYMMDD>[-revN]/`
40
+
41
+ 规则如下:
42
+
43
+ - `model_id`:必须与 `configs/models.yaml` 中的模型 id 一致。
44
+ - `scope`:推荐使用 `standard`、`basic-education`、`full`、`safety`、`highlevel`、`general`、`custom-<slug>`。
45
+ - `YYYYMMDD`:结果定稿日,不强制等于任务启动日。
46
+ - 同一天同一范围如果重新定稿,追加 `-rev2`、`-rev3`。
47
+ - 真实运行时使用的 `run_id` 不要求和结果包目录名一致,但必须写进 `manifest.json`。
48
+
49
+ ## 5. 结果包内容规范
50
+
51
+ ### 5.1 必备文件
52
+
53
+ 每个正式结果包至少包含:
54
+
55
+ - `README.md`
56
+ - `manifest.json`
57
+ - `summary.json`
58
+
59
+ ### 5.2 可选文件
60
+
61
+ 可按需增加:
62
+
63
+ - `notes.md`
64
+ - `artifacts/`
65
+
66
+ `artifacts/` 只允许放精选的小文件,例如手工整理后的表格、图表、失败样例摘录,不允许把运行时全量产物整体搬进来。
67
+
68
+ ### 5.3 `README.md` 最少字段
69
+
70
+ 每个结果包内的 `README.md` 至少写清楚:
71
+
72
+ - `run_id`
73
+ - `model_id`
74
+ - `scope`
75
+ - `finished_at`
76
+ - `source_outputs`
77
+ - `total_judged`
78
+ - `total_failures`
79
+ - 已知 caveat 或未完成项
80
+
81
+ ### 5.4 `manifest.json` 必备字段
82
+
83
+ 每个结果包内的 `manifest.json` 至少包含:
84
+
85
+ - `schema_version`
86
+ - `result_id`
87
+ - `run_id`
88
+ - `model_id`
89
+ - `provider_name`
90
+ - `benchmark_scope`
91
+ - `finished_at`
92
+ - `git_commit`
93
+ - `source_outputs`
94
+ - `included_files`
95
+ - `excluded_artifacts`
96
+
97
+ 可选字段:
98
+
99
+ - `base_url`
100
+ - `judge_model_ids`
101
+ - `notes`
102
+
103
+ ## 6. 禁止提交的内容
104
+
105
+ 以下内容默认禁止进入 `results/`:
106
+
107
+ - `outputs/` 下的任何目录
108
+ - 全量 `raw_responses/`
109
+ - `*.checkpoint.json`
110
+ - `*.retries.jsonl`
111
+ - `*.failures.jsonl`
112
+ - `*.log`
113
+ - 密钥文件、环境变量导出、任何 secret
114
+ - `smoke-*`、`probe-*`、`tmp-*`、`debug-*` 之类的试跑结果
115
+
116
+ `judged_results/*.jsonl` 默认也不提交。只有在它本身就是正式交付物的一部分,而且单文件不超过 `5 MB` 时,才允许作为 `artifacts/` 的受控附件入库,并且必须在该结果包的 `README.md` 里说明原因。
117
+
118
+ ## 7. 大小与稳定性要求
119
+
120
+ - 单个结果包目标大小:`<= 2 MB`
121
+ - 没有明确理由时,不得超过 `20 MB`
122
+ - `summary.json` 一经提交,就视为该次结果的 canonical snapshot
123
+ - 如果只修正文案,可以在原目录内修改
124
+ - 如果指标、范围、样本集或统计口径发生变化,必须新建 `-revN` 目录,而不是静默覆盖旧结果
125
+
126
+ ## 8. 清理策略
127
+
128
+ - `outputs/` 里的 `smoke-*`、`probe-*`、`tmp-*`、`debug-*` 可随时本地删除。
129
+ - 已经升格到 `results/` 的 run,其 `outputs/` 只作为本地审计缓存,不再作为仓库正式资产。
130
+ - 需要长期保留的全量原始产物,应转存到仓库外归档位置;`manifest.json` 只记录来源路径,不把大文件带入 Git。
131
+
132
+ ## 9. 标准目录模板
133
+
134
+ ```text
135
+ results/
136
+ ├── README.md
137
+ └── innospark-235b-standard-20260424/
138
+ ├── README.md
139
+ ├── manifest.json
140
+ ├── summary.json
141
+ └── artifacts/ # optional
142
+ ```
143
+
144
+ ## 10. 当前落地约定
145
+
146
+ - `outputs/` 继续保持 Git ignore。
147
+ - 当前首个标准样例是 `results/innospark-235b-standard-20260424/`。
148
+ - 后续所有正式评测结果都按本规范落入 `results/`,不再直接把运行时垃圾留给 Git 视图。
docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ELBench 团队维护指南
2
+
3
+ 本指南面向 ELBench 团队维护者,目标是把“什么应该保留、什么应该删、什么地方负责什么”说清楚,减少仓库熵,保证多人协作和大模型协作时都不会把项目越改越乱。
4
+
5
+ ## 1. 总原则
6
+
7
+ - 配置优先。新增数据、模型、judge 时,优先改注册表和配置,不直接侵入主流程。
8
+ - 边界清晰。加载、执行、判分、持久化、汇总必须分层,不要混写。
9
+ - 输出稳定。`outputs/` 结构、结果字段、日志字段不能随意改。
10
+ - 数据归位。正式 benchmark 数据统一放在 `data/benchmark_root/` 下,不能散落到根目录或外挂目录。
11
+ - 数据集不删。任何 benchmark 数据集文件都不得因为“暂未接入”“暂未注册”“暂时无测试”而删除;未接入的数据集只能保留并标记为待接入。
12
+ - 轻仓库。缓存、临时分析文档、历史脚本、系统垃圾文件不要留在主仓库。
13
+
14
+ 一句话:**优先走“数据目录 -> 注册表 -> 字段映射 -> judge 路由 -> 边界层代码 -> 测试验证”这条路径。**
15
+
16
+ ## 2. 当前清理与重构结论
17
+
18
+ 这次整理后,以下判断作为当前团队共识:
19
+
20
+ | 路径 | 结论 | 原因 |
21
+ |---|---|---|
22
+ | `tests/` | 保留 | 这是正式回归保障,不是临时目录 |
23
+ | `src/elbench/basic_education_runtime/` | 保留 | `基本教育` 多轮桥接的内置运行时 |
24
+ | `third_party/` | 删除 | 基本教育运行时已内收进项目内部,不再作为外挂目录存在 |
25
+ | `run.sh` / `setup.sh` | 删除 | 过时、未被文档和主流程使用 |
26
+ | `docs/*analysis*.md` | 删除 | 历史分析稿,且已过时 |
27
+ | `docs/sample_command.md` | 删除 | 使用了已废弃的模型配置 |
28
+ | `edu_eval_spec_for_codex.md` | 删除 | 早期实现规格,已被当前代码和 README 替代 |
29
+ | `.DS_Store`、`pytest-cache-files-*`、`__pycache__`、`outputs/` | 不提交,按需本地删除 | 运行时或系统垃圾文件 |
30
+
31
+ 额外说明:
32
+
33
+ - ELBench 当前只有 `src/elbench/` 是项目源码根。
34
+ - `基本教育` 的内部运行时模块位于 `src/elbench/basic_education_runtime/`。
35
+ - 这个内部运行时来源于 ELMES,但现在已经按 ELBench 自有模块维护,不再以外挂目录的形式存在。
36
+
37
+ ## 3. 顶层目录/文件职责表
38
+
39
+ | 路径 | 是否应长期保留 | 作用 | 什么时候改 |
40
+ |---|---|---|---|
41
+ | `README.md` | 保留 | 英文默认首页,对外说明项目能力、结构、安装与运行方法 | 仓库能力、结构、安装方式变化时 |
42
+ | `README.zh-CN.md` | 保留 | 中文说明 | 与英文 README 同步更新时 |
43
+ | `docs/TEAM_MAINTENANCE_GUIDE.zh-CN.md` | 保留 | 团队内部维护规范和目录职责表 | 维护规范变化时 |
44
+ | `pyproject.toml` | 保留 | 包定义、依赖、可选 extras、CLI 入口 | 增加依赖、调整安装方式时 |
45
+ | `.gitignore` | 保留 | 忽略缓存、输出和系统垃圾文件 | 新增运行时垃圾目录时 |
46
+ | `configs/` | 保留 | 全部配置驱动入口 | 新增模块、文件、模型、judge、并发规则时 |
47
+ | `data/benchmark_root/` | 保留 | 正式 benchmark 数据根目录 | 新增、整理 benchmark 数据时 |
48
+ | `docs/` | 保留,但只留活文档 | 团队文档目录,不放一次性分析稿 | 文档新增或重写时 |
49
+ | `scripts/run_benchmark.py` | 保留 | 运行入口薄封装 | 很少改;只改入口行为 |
50
+ | `src/elbench/` | 保留 | ELBench 正式源码 | 根据职责分层修改 |
51
+ | `tests/` | 保留 | 回归测试与 smoke test | 新增/修改功能后同步补测试 |
52
+ | `results/` | 保留 | 正式结果包目录,只放审核后的可提交结果 | 新增正式结果快照时 |
53
+ | `outputs/` | 不入库 | 运行产物目录 | 本地跑实验时自动生成 |
54
+
55
+ 正式结果包规范见 `docs/RESULT_STORAGE_SPEC.zh-CN.md`,当前结果索引见 `results/README.md`。
56
+
57
+ ## 4. `configs/` 职责表
58
+
59
+ | 路径 | 作用 | 注意事项 |
60
+ |---|---|---|
61
+ | `configs/app.yaml` | 全局默认配置,如 data root、output root、默认并发、默认 retry | 默认最大并发当前为 `100` |
62
+ | `configs/module_registry.yaml` | 模块注册表 | 这里只定义模块,不定义文件 |
63
+ | `configs/file_registry.yaml` | benchmark 文件注册表 | 文件是否“正式接入”以这里为准 |
64
+ | `configs/field_mappings.yaml` | 原始字段到统一 schema 的映射 | 字段变了优先改这里 |
65
+ | `configs/providers.yaml` | provider 注册与默认能力 | 不放具体模型实例 |
66
+ | `configs/models.yaml` | 被测模型和 judge 模型实例配置 | 单模型超时、限流、provider kwargs 在这里配 |
67
+ | `configs/judges.yaml` | task 到 rule / llm judge 的路由 | judge 路由唯一入口 |
68
+ | `configs/basic_education.yaml` | 基本教育桥接执行配置 | 只管执行,不替代 benchmark 数据注册 |
69
+
70
+ ## 5. `src/elbench/` 代码职责表
71
+
72
+ | 路径 | 作用 | 什么时候改 |
73
+ |---|---|---|
74
+ | `src/elbench/cli.py` | CLI 参数入口 | 新增命令行参数时 |
75
+ | `src/elbench/config/loader.py` | 配置加载 | 新增配置文件或 schema 时 |
76
+ | `src/elbench/schemas/config.py` | 配置 schema | 配置结构变更时 |
77
+ | `src/elbench/schemas/evaluation.py` | Sample / Result / Judge schema | 改输出契约时,需极慎重 |
78
+ | `src/elbench/registry/file_registry.py` | 文件发现与解析 | 文件定位逻辑变化时 |
79
+ | `src/elbench/loaders/base.py` | loader 抽象基类 | loader 接口变化时 |
80
+ | `src/elbench/loaders/jsonl_loader.py` | JSONL 加载 | JSONL 读取逻辑变更时 |
81
+ | `src/elbench/loaders/xlsx_loader.py` | XLSX 加载 | Excel 读取逻辑变更时 |
82
+ | `src/elbench/loaders/basic_education_yaml_loader.py` | 基本教育 YAML 样本加载 | 基本教育数据模板解析变化时 |
83
+ | `src/elbench/loaders/normalizer.py` | 统一 sample 归一化 | Sample 构造规则变化时 |
84
+ | `src/elbench/loaders/resolvers.py` | 派生字段解析,如 `dimension` | 某些字段需要特殊派生逻辑时 |
85
+ | `src/elbench/providers/base.py` | 模型客户端统一接口 | provider 抽象变化时 |
86
+ | `src/elbench/providers/factory.py` | provider 工厂 | 新增 provider 类时 |
87
+ | `src/elbench/providers/openai_compatible.py` | OpenAI 兼容 provider | 接 OpenAI-compatible 厂商时 |
88
+ | `src/elbench/providers/mock.py` | 本地 mock provider | 本地 smoke 或 judge 回归时 |
89
+ | `src/elbench/execution/runner.py` | 主执行器 | 执行调度变更时,慎改 |
90
+ | `src/elbench/execution/basic_education.py` | 基本教育多轮桥接 | 调整内部运行时调用、结果导入、恢复逻辑时 |
91
+ | `src/elbench/execution/rate_limit.py` | 并发与限流控制 | 限流策略变化时 |
92
+ | `src/elbench/execution/retry.py` | 重试与退避 | 重试策略变化时 |
93
+ | `src/elbench/judges/router.py` | judge 路由 | task 到 judge 映射变化时 |
94
+ | `src/elbench/judges/base.py` | judge 抽象基类 | judge 接口变化时 |
95
+ | `src/elbench/judges/llm_judge.py` | `LLM-as-a-Judge` 执行器 | 主观题 judge 链路升级时 |
96
+ | `src/elbench/judges/llm_prompting.py` | judge prompt 模板 | rubric 和 judge prompt 调整时 |
97
+ | `src/elbench/judges/judge_safety.py` | 安全可信相关判分 | 安全任务判分逻辑变化时 |
98
+ | `src/elbench/judges/judge_teaching_harm.py` | 教学安全客观题判分 | SATAs 等规则判分变化时 |
99
+ | `src/elbench/judges/judge_highlevel.py` | 高阶育人判分 | `highlevel_edu` / `highlevel_omni` 变化时 |
100
+ | `src/elbench/judges/judge_mmlu.py` | MMLU-Pro / C-Eval 判分 | 通用客观题扩展时 |
101
+ | `src/elbench/judges/judge_ifeval.py` | IFEval 判分 | 指令遵循判分变化时 |
102
+ | `src/elbench/judges/judge_math.py` | Math-500 / AIME 判分 | 数学题判分变化时 |
103
+ | `src/elbench/basic_education_runtime/` | 基本教育内部多轮运行时 | 修改运行时模型编排、导出、评估逻辑时 |
104
+ | `src/elbench/persistence/writers.py` | 结果写盘 | 输出结构变化时 |
105
+ | `src/elbench/persistence/checkpoint.py` | checkpoint 持久化 | 断点续跑行为变化时 |
106
+ | `src/elbench/persistence/logging.py` | 日志初始化 | 日志格式或级别调整时 |
107
+ | `src/elbench/summary/aggregator.py` | 汇总统计 | 新增 summary 维度时 |
108
+ | `src/elbench/utils/parsing.py` | 通用解析工具 | 文本/答案解析工具扩展时 |
109
+
110
+ ## 6. `tests/` 职责表
111
+
112
+ | 路径 | 作用 | 什么时候必须更新 |
113
+ |---|---|---|
114
+ | `tests/test_registry_and_loaders.py` | 校验注册表是否能解析当前 bench 文件,以及 loader 是否能产出统一 sample | 新增 benchmark 文件、调整 loader、改 field mapping 后 |
115
+ | `tests/test_judges.py` | 校验各类客观题和 `LLM-as-a-Judge` 路径的基本行为 | 改 judge、改 prompt 解析、改客观题判分逻辑后 |
116
+ | `tests/test_basic_education_bridge.py` | 校验基本教育 45 题配置、场景任务计数和内部运行时结果导入辅助逻辑 | 改基本教育配置或桥接实现后 |
117
+
118
+ ## 7. `src/elbench/basic_education_runtime/` 保留范围
119
+
120
+ 这个目录是 `基本教育` 的内置运行时,不是独立仓库。
121
+
122
+ 当前应保留:
123
+
124
+ - `src/elbench/basic_education_runtime/` 下的源码
125
+ - `src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf`
126
+
127
+ 当前不应再出现:
128
+
129
+ - 外挂式旧 third-party 运行时目录
130
+ - 与 ELBench 无关的上游示例、CI、锁文件
131
+ - 单独的外部项目说明口径
132
+
133
+ ## 8. 新增评测数据的标准流程
134
+
135
+ 在任何“新增/待接入/重构”场景下,都适用一条硬规则:
136
+
137
+ - 不要删除任何 benchmark 数据集文件。未接入的数据集要保留在 `data/benchmark_root/` 下,并通过文档或注册状态明确标记为“待接入”。
138
+
139
+ ### 场景 A:新增文件,但仍属于现有模块
140
+
141
+ 1. 把数据放到 `data/benchmark_root/` 下正确的模块目录。
142
+ 2. 在 `configs/file_registry.yaml` 注册文件。
143
+ 3. 在 `configs/field_mappings.yaml` 增加字段映射。
144
+ 4. 在 `configs/judges.yaml` 增加或调整 judge 路由。
145
+ 5. 如需派生字段逻辑,在 `src/elbench/loaders/resolvers.py` 扩展。
146
+ 6. 客观题优先规则判分,主��题再走 `LLM-as-a-Judge`。
147
+ 7. 补至少一条测试。
148
+ 8. 跑 `inspect` 和单测。
149
+
150
+ ### 场景 B:新增一个全新模块
151
+
152
+ 1. 在 `configs/module_registry.yaml` 注册模块。
153
+ 2. 在 `data/benchmark_root/` 下建立正式目录。
154
+ 3. 为该模块每个文件补 `file_registry.yaml`。
155
+ 4. 为该模块每个文件补 `field_mappings.yaml`。
156
+ 5. 为该模块 task 补 `judges.yaml`。
157
+ 6. 只有配置表达不了时,才新增 loader、judge 或执行逻辑。
158
+
159
+ 补充:`基本教育` 不是数据组织例外。它的数据也必须放在 `data/benchmark_root/基本教育/`,只是运行时要通过 `src/elbench/execution/basic_education.py` 调用内部 `basic_education_runtime`。
160
+
161
+ ## 9. 什么时候优先改配置,什么时候改代码
162
+
163
+ ### 优先改配置
164
+
165
+ - 文件路径变化
166
+ - 文件归属模块或子集变化
167
+ - 原始字段名变化
168
+ - task 从 `rule` 切到 `llm`
169
+ - 模型 timeout / retry / rate limit 变化
170
+
171
+ ### 必须改代码
172
+
173
+ - 数据格式不是现有 loader 能覆盖的格式
174
+ - 字段派生逻辑无法由映射表达
175
+ - 新 provider API 形态与现有 adapter 不兼容
176
+ - judge 逻辑本身是新的
177
+ - summary 要新增新的聚合方式
178
+
179
+ ### 修改 `runner.py` 的前提
180
+
181
+ 只有当配置、adapter、judge、loader 都不能表达需求时,才允许改 `src/elbench/execution/runner.py`。改完必须补测试并跑 smoke run。
182
+
183
+ ## 10. judge 维护规范
184
+
185
+ ### 客观题
186
+
187
+ 客观题默认优先规则判分,适用于:
188
+
189
+ - 单选题
190
+ - 多选题
191
+ - 判断题
192
+ - 有稳定标准答案的结构化任务
193
+
194
+ 优先修改位置:
195
+
196
+ - `src/elbench/judges/judge_teaching_harm.py`
197
+ - `src/elbench/judges/judge_highlevel.py`
198
+ - `src/elbench/judges/judge_mmlu.py`
199
+ - `src/elbench/judges/judge_ifeval.py`
200
+ - `src/elbench/judges/judge_math.py`
201
+ - `src/elbench/utils/parsing.py`
202
+
203
+ ### 主观题
204
+
205
+ 主观题优先 `LLM-as-a-Judge`,且 judge model 必须和被测模型解耦。
206
+
207
+ 优先修改位置:
208
+
209
+ - `configs/judges.yaml`
210
+ - `src/elbench/judges/llm_prompting.py`
211
+ - `src/elbench/judges/llm_judge.py`
212
+
213
+ 禁止:
214
+
215
+ - 让被测模型给自己打分
216
+ - 把 judge prompt 写死在 `runner.py`
217
+ - 把 rubric 散落到无关文件
218
+
219
+ ## 11. 模型与 provider 接入规范
220
+
221
+ ### 新增模型
222
+
223
+ 1. 先确认现有 adapter 是否已支持。
224
+ 2. 只改 `configs/models.yaml`。
225
+ 3. 如需新的限流或 timeout,也只在该模型配置里加。
226
+
227
+ ### 新增 provider
228
+
229
+ 1. 在 `src/elbench/providers/` 新增 adapter。
230
+ 2. 在 `factory.py` 注册。
231
+ 3. 在 `configs/providers.yaml` 增加 provider。
232
+ 4. 在 `configs/models.yaml` 增加具体模型实例。
233
+
234
+ 要求:
235
+
236
+ - provider 差异必须封装在 adapter 内
237
+ - 不允许在 runner 或 judge 中写 provider-specific 参数名
238
+
239
+ ## 12. 与大模型协作维护的要求
240
+
241
+ 每次让大模型改 ELBench 时,至少要明确告诉它:
242
+
243
+ - 项目名是 `ELBench`
244
+ - 这是长期维护 benchmark 框架,不是 demo
245
+ - 正式数据根目录是 `data/benchmark_root/`
246
+ - 当前 judge 策略是“客观题 rule,主观题 llm judge”
247
+ - 先看 `README.md`、本指南和相关配置,再改代码
248
+
249
+ 并要求它:
250
+
251
+ 1. 优先改配置
252
+ 2. 新增数据时同步改 `file_registry`、`field_mappings`、`judges`
253
+ 3. 改 judge 时补测试
254
+ 4. 提交前跑最小验证命令
255
+
256
+ ## 13. 提交前检查清单
257
+
258
+ - [ ] 新数据是否放在 `data/benchmark_root/` 正确目录
259
+ - [ ] `configs/file_registry.yaml` 是否已同步更新
260
+ - [ ] `configs/field_mappings.yaml` 是否已同步更新
261
+ - [ ] `configs/judges.yaml` 是否已同步更新
262
+ - [ ] 是否误改输出结构
263
+ - [ ] 是否新增或更新了测试
264
+ - [ ] 是否清掉了 `outputs/`、缓存目录、系统垃圾文件
265
+ - [ ] 是否执行了:
266
+ - `python -m unittest tests.test_basic_education_bridge tests.test_registry_and_loaders tests.test_judges`
267
+ - `python scripts/run_benchmark.py inspect`
268
+ - 至少一次小样本 `run`
269
+
270
+ ## 14. 推荐验证命令
271
+
272
+ ```bash
273
+ python -m unittest tests.test_basic_education_bridge tests.test_registry_and_loaders tests.test_judges
274
+ python scripts/run_benchmark.py inspect
275
+ python scripts/run_benchmark.py run --model-id mock.default --max-samples 3 --run-id smoke-test --no-resume
276
+ ```
277
+
278
+ ## 15. 常见错误
279
+
280
+ ### 错误 1:只改代码,不改注册表
281
+
282
+ 后果:文件不会被主流程识别,`inspect` 看不到。
283
+
284
+ ### 错误 2:字段名变了,直接改 loader
285
+
286
+ 正确做法:先看能不能只改 `configs/field_mappings.yaml`。
287
+
288
+ ### 错误 3:新增模型时去改 runner
289
+
290
+ 正确做法:优先改 `configs/models.yaml`,必要时才新增 adapter。
291
+
292
+ ### 错误 4:把一次性分析稿留在 `docs/`
293
+
294
+ 正确做法:只保留活文档;临时分析不要长期入库。
295
+
296
+ ### 错误 5:把 `outputs/`、缓存或 `.DS_Store` 提交进仓库
297
+
298
+ 正确做法:确保它们继续被 `.gitignore` 忽略,并在提交前清理工作区。
docs/benchmark_status_2026-05-25.md ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Benchmark Status 2026-05-25
2
+
3
+ ## Current Scope
4
+
5
+ - Cheap relay models currently evaluated without `安全可信`
6
+ - Active modules for relay runs:
7
+ - `通用模型`
8
+ - `高阶育人`
9
+ - `基本教育`
10
+ - Reason:
11
+ - InnoSpark relay may pre-filter safety prompts, which makes `安全可信` unfair for official comparison
12
+
13
+ ## Endpoint Notes
14
+
15
+ - Cheap relay models:
16
+ - `https://api.innospark.cn/v1`
17
+ - key env: `INNOSPARK_RELAY_API_KEY`
18
+ - GPT/Claude support endpoint:
19
+ - `http://35.220.164.252:3888/v1`
20
+ - key env: `EXTERNAL_SUPPORT_API_KEY`
21
+ - Judge / evaluate / student endpoint:
22
+ - current `qwen-test` notebook `9501` endpoint became unstable and later returned `404`
23
+ - this must be repaired or replaced before further official judged runs
24
+
25
+ ## Officially Usable Results
26
+
27
+ - `gpt-5.4`
28
+ - full official run completed cleanly
29
+ - run id: `full-gpt-5.4-20260519-v2`
30
+ - `gemini-3-flash-preview`
31
+ - `no-safety` relay run completed
32
+ - run id: `fair-gemini-3-flash-preview-nosafety-20260521-v1`
33
+ - `deepseek-v3.2`
34
+ - `no-safety` relay run mostly completed
35
+ - run id: `fair-deepseek-v3.2-nosafety-20260520-v1`
36
+ - still worth a final cleanup check before treating as final
37
+
38
+ ## Dirty / Incomplete Runs
39
+
40
+ - `gpt-5.2-pro`
41
+ - run id: `full-gpt-5.2-pro-20260520-v1`
42
+ - interrupted by quota exhaustion
43
+ - `doubao-seed-2-0-pro-260215`
44
+ - run id: `fair-doubao-seed-2-0-pro-260215-nosafety-20260521-v1`
45
+ - interrupted by local checkpoint write `PermissionError`
46
+ - `kimi-k2.6`
47
+ - runs:
48
+ - `fair-kimi-k2.6-nosafety-20260521-v1`
49
+ - `fair-kimi-k2.6-nosafety-20260521-v2`
50
+ - `fair-kimi-k2.6-nosafety-20260521-v3`
51
+ - key findings:
52
+ - relay has hard rate limit around `1 request/second`
53
+ - model-specific throttling was added: `qps: 1`, `max_concurrency: 1`
54
+ - later hit quota exhaustion, then resume attempts became messy
55
+ - checkpoint showed many completed items, but `raw/judged` artifacts were not present
56
+ - conclusion:
57
+ - current Kimi runs are not directly salvageable as official final results
58
+ - `deepseek-r1-250528`
59
+ - official run:
60
+ - `fair-deepseek-r1-250528-nosafety-20260524-v1`
61
+ - probe run:
62
+ - `probe-deepseek-r1-250528-nosafety-20260525-c2t480-v1`
63
+ - key findings:
64
+ - high `ReadTimeout` rate under prior settings
65
+ - later probe failures were caused by broken `qwen-test` judge endpoint `404`
66
+ - conclusion:
67
+ - not usable as final result yet
68
+
69
+ ## Remaining Models To Finish Cleanly
70
+
71
+ - `doubao-seed-2-0-pro-260215`
72
+ - `kimi-k2.6`
73
+ - `deepseek-r1-250528`
74
+ - `gemini-3.1-pro-preview`
75
+ - optional later if quota allows:
76
+ - `gpt-5.2-pro`
77
+
78
+ ## Important Technical Findings
79
+
80
+ - `kimi-k2.6` needs special low-rate execution
81
+ - `deepseek-r1-250528` likely needs lower concurrency and longer timeout than the default cheap-model settings
82
+ - judge endpoint health must be verified before any further official judged run
83
+ - for any future run, do not trust checkpoint alone
84
+ - future official runs must confirm all of these are being written:
85
+ - `outputs/raw_responses/...`
86
+ - `outputs/judged_results/...`
87
+ - `outputs/logs/...`
88
+ - `outputs/summaries/...`
89
+
90
+ ## Next Workstream
91
+
92
+ - Repair or replace the `qwen-test` judge endpoint
93
+ - Run a full-chain smoke test
94
+ - Start organizing completed clean results into:
95
+ - standardized result folders
96
+ - report tables
97
+ - leaderboard
pyproject.toml ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "elbench"
7
+ version = "0.1.0"
8
+ description = "Config-driven benchmark framework for education LLM evaluation."
9
+ requires-python = ">=3.11"
10
+ dependencies = [
11
+ "httpx>=0.27.0",
12
+ "nltk>=3.8",
13
+ "latex2sympy2_extended>=1.10.1",
14
+ "openpyxl>=3.1.0",
15
+ "pylatexenc>=2.10",
16
+ "pydantic>=2.7.0",
17
+ "PyYAML>=6.0.0",
18
+ "sympy>=1.13.0",
19
+ "word2number>=1.1",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ basic-education = [
24
+ "aiosqlite>=0.20.0",
25
+ "click>=8.1.0",
26
+ "grandalf>=0.8",
27
+ "langchain>=0.3.0",
28
+ "langchain-openai>=0.3.0",
29
+ "langgraph>=0.4.0",
30
+ "langgraph-checkpoint-sqlite>=2.0.0",
31
+ "matplotlib>=3.8.0",
32
+ "pandas>=2.2.0",
33
+ "polyfactory>=2.16.0",
34
+ "pyppeteer>=2.0.0",
35
+ "tenacity>=9.0.0",
36
+ "tqdm>=4.66.0",
37
+ "typing-extensions>=4.12.0",
38
+ ]
39
+ dev = [
40
+ "pytest>=8.0.0",
41
+ ]
42
+ dashboard = [
43
+ "gradio==6.4.0",
44
+ "plotly>=5.20.0",
45
+ ]
46
+
47
+ [project.scripts]
48
+ elbench = "elbench.cli:main"
49
+
50
+ [tool.setuptools]
51
+ package-dir = {"" = "src"}
52
+
53
+ [tool.setuptools.packages.find]
54
+ where = ["src"]
55
+
56
+ [tool.setuptools.package-data]
57
+ "elbench.basic_education_runtime.assets.fonts" = ["*.ttf"]
58
+ "elbench.dashboard" = ["*.json"]
requirements.txt ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Hugging Face Spaces installs this file before copying /app.
2
+ # Gradio, spaces, and huggingface_hub are preinstalled by the Spaces runtime.
3
+ httpx>=0.27.0
4
+ nltk>=3.8
5
+ latex2sympy2_extended>=1.10.1
6
+ openpyxl>=3.1.0
7
+ pylatexenc>=2.10
8
+ pydantic>=2.7.0
9
+ PyYAML>=6.0.0
10
+ sympy>=1.13.0
11
+ word2number>=1.1
12
+ aiosqlite>=0.20.0
13
+ langchain>=0.3.0,<0.4.0
14
+ langchain-openai>=0.3.0,<0.4.0
15
+ langgraph>=0.4.0,<0.7.0
16
+ langgraph-checkpoint-sqlite>=2.0.0,<3.0.0
17
+ matplotlib>=3.8.0
18
+ pandas>=2.2.0
19
+ plotly>=5.20.0
20
+ polyfactory>=2.16.0
21
+ tenacity>=9.0.0
22
+ tqdm>=4.66.0
23
+ typing-extensions>=4.12.0
24
+ websockets>=12.0,<14.0
scripts/build_stage_report.py ADDED
@@ -0,0 +1,423 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import json
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+
10
+ ROOT = Path(__file__).resolve().parents[1]
11
+ OUTPUT_DIR = ROOT / "results" / "stage-report-20260525"
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class RunSpec:
16
+ model_id: str
17
+ run_id: str
18
+ summary_relpath: str
19
+ scope: str
20
+ included_modules: tuple[str, ...]
21
+ excluded_modules: tuple[str, ...]
22
+ notes: tuple[str, ...] = ()
23
+
24
+
25
+ RUN_SPECS = (
26
+ RunSpec(
27
+ model_id="gpt-5.4",
28
+ run_id="full-gpt-5.4-20260519-v2",
29
+ summary_relpath="outputs/summaries/full-gpt-5.4-20260519-v2/gpt-5.4.summary.json",
30
+ scope="full_scope",
31
+ included_modules=("general", "safety", "highlevel", "basic_education"),
32
+ excluded_modules=(),
33
+ notes=("cleanest currently available full-scope result",),
34
+ ),
35
+ RunSpec(
36
+ model_id="deepseek-v3.2",
37
+ run_id="fair-deepseek-v3.2-nosafety-20260520-v1",
38
+ summary_relpath="outputs/summaries/fair-deepseek-v3.2-nosafety-20260520-v1/deepseek-v3.2.summary.json",
39
+ scope="relay_no_safety",
40
+ included_modules=("general", "highlevel", "basic_education"),
41
+ excluded_modules=("safety",),
42
+ notes=("no-safety relay run; final cleanup still recommended.",),
43
+ ),
44
+ )
45
+
46
+
47
+ GENERAL_TASKS = (
48
+ "mmlu_pro",
49
+ "ceval",
50
+ "ifeval",
51
+ "math_500",
52
+ "aime24",
53
+ "aime25",
54
+ "aime26",
55
+ "gsm8k",
56
+ )
57
+ HIGHLEVEL_TASKS = ("highlevel_edu", "highlevel_omni")
58
+ SAFETY_TASKS = (
59
+ "safety_refusal",
60
+ "safety_guidance",
61
+ "safety_answer",
62
+ "teaching_harm",
63
+ "adversarial_safety",
64
+ )
65
+ BASIC_TASKS = (
66
+ "basic_knowledge_point_explanation",
67
+ "basic_contextualized_question_generation",
68
+ "basic_interdisciplinary_lesson_plan_generation",
69
+ "basic_guided_problem_solving_teaching",
70
+ )
71
+ BASIC_TASK_MAX_SCORE = {
72
+ "basic_knowledge_point_explanation": 10.0,
73
+ "basic_contextualized_question_generation": 5.0,
74
+ "basic_interdisciplinary_lesson_plan_generation": 5.0,
75
+ # Guided task should use pass_rate, not avg_score.
76
+ "basic_guided_problem_solving_teaching": 1.0,
77
+ }
78
+
79
+
80
+ def load_json(path: Path) -> dict[str, Any]:
81
+ return json.loads(path.read_text(encoding="utf-8"))
82
+
83
+
84
+ def format_pct(value: float | None) -> str:
85
+ if value is None:
86
+ return ""
87
+ return f"{value * 100:.2f}"
88
+
89
+
90
+ def format_num(value: float | None) -> str:
91
+ if value is None:
92
+ return ""
93
+ return f"{value:.6f}".rstrip("0").rstrip(".")
94
+
95
+
96
+ def weighted_pass_rate(summary: dict[str, Any], task_names: tuple[str, ...]) -> float | None:
97
+ by_task = summary.get("by_task", {})
98
+ total_count = 0
99
+ weighted_sum = 0.0
100
+ for task in task_names:
101
+ task_data = by_task.get(task)
102
+ if not task_data:
103
+ continue
104
+ count = int(task_data.get("count", 0) or 0)
105
+ pass_rate = task_data.get("pass_rate")
106
+ if count <= 0 or pass_rate is None:
107
+ continue
108
+ total_count += count
109
+ weighted_sum += count * float(pass_rate)
110
+ if total_count == 0:
111
+ return None
112
+ return weighted_sum / total_count
113
+
114
+
115
+ def weighted_basic_score(summary: dict[str, Any]) -> float | None:
116
+ by_task = summary.get("by_task", {})
117
+ total_count = 0
118
+ weighted_sum = 0.0
119
+ for task in BASIC_TASKS:
120
+ task_data = by_task.get(task)
121
+ if not task_data:
122
+ continue
123
+ count = int(task_data.get("count", 0) or 0)
124
+ if count <= 0:
125
+ continue
126
+ if task == "basic_guided_problem_solving_teaching":
127
+ normalized = float(task_data.get("pass_rate", 0.0) or 0.0)
128
+ else:
129
+ max_score = BASIC_TASK_MAX_SCORE[task]
130
+ normalized = float(task_data.get("avg_score", 0.0) or 0.0) / max_score
131
+ total_count += count
132
+ weighted_sum += count * normalized
133
+ if total_count == 0:
134
+ return None
135
+ return weighted_sum / total_count
136
+
137
+
138
+ def scope_score(row: dict[str, Any]) -> float | None:
139
+ parts: list[float] = []
140
+ if row["general_score"] is not None:
141
+ parts.append(row["general_score"])
142
+ if row["highlevel_score"] is not None:
143
+ parts.append(row["highlevel_score"])
144
+ if row["basic_score"] is not None:
145
+ parts.append(row["basic_score"])
146
+ if row["scope"] == "full_scope" and row["safety_score"] is not None:
147
+ parts.append(row["safety_score"])
148
+ if not parts:
149
+ return None
150
+ return sum(parts) / len(parts)
151
+
152
+
153
+ def build_rows() -> list[dict[str, Any]]:
154
+ rows: list[dict[str, Any]] = []
155
+ for spec in RUN_SPECS:
156
+ summary_path = ROOT / spec.summary_relpath
157
+ summary = load_json(summary_path)
158
+ row = {
159
+ "model_id": spec.model_id,
160
+ "run_id": spec.run_id,
161
+ "scope": spec.scope,
162
+ "summary_path": spec.summary_relpath.replace("\\", "/"),
163
+ "included_modules": ", ".join(spec.included_modules),
164
+ "excluded_modules": ", ".join(spec.excluded_modules),
165
+ "notes": " | ".join(spec.notes),
166
+ "total_judged": int(summary.get("total_judged", 0) or 0),
167
+ "total_failures": int(summary.get("total_failures", 0) or 0),
168
+ "general_score": weighted_pass_rate(summary, GENERAL_TASKS),
169
+ "highlevel_score": weighted_pass_rate(summary, HIGHLEVEL_TASKS),
170
+ "safety_score": weighted_pass_rate(summary, SAFETY_TASKS),
171
+ "basic_score": weighted_basic_score(summary),
172
+ }
173
+ row["stage_score"] = scope_score(row)
174
+ rows.append(row)
175
+ return rows
176
+
177
+
178
+ def write_csv(path: Path, rows: list[dict[str, Any]], fieldnames: list[str]) -> None:
179
+ path.parent.mkdir(parents=True, exist_ok=True)
180
+ with path.open("w", encoding="utf-8", newline="") as fh:
181
+ writer = csv.DictWriter(fh, fieldnames=fieldnames, extrasaction="ignore")
182
+ writer.writeheader()
183
+ for row in rows:
184
+ writer.writerow(row)
185
+
186
+
187
+ def make_overall_table(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
188
+ table = []
189
+ for row in rows:
190
+ table.append(
191
+ {
192
+ "model_id": row["model_id"],
193
+ "scope": row["scope"],
194
+ "run_id": row["run_id"],
195
+ "total_judged": row["total_judged"],
196
+ "total_failures": row["total_failures"],
197
+ "general_score_pct": format_pct(row["general_score"]),
198
+ "highlevel_score_pct": format_pct(row["highlevel_score"]),
199
+ "safety_score_pct": format_pct(row["safety_score"]),
200
+ "basic_score_pct": format_pct(row["basic_score"]),
201
+ "stage_score_pct": format_pct(row["stage_score"]),
202
+ "included_modules": row["included_modules"],
203
+ "excluded_modules": row["excluded_modules"],
204
+ "notes": row["notes"],
205
+ }
206
+ )
207
+ return table
208
+
209
+
210
+ def make_scope_leaderboard(rows: list[dict[str, Any]], scope: str) -> list[dict[str, Any]]:
211
+ filtered = [row for row in rows if row["scope"] == scope]
212
+ filtered.sort(key=lambda item: (item["stage_score"] is None, -(item["stage_score"] or 0.0), item["model_id"]))
213
+ leaderboard = []
214
+ for index, row in enumerate(filtered, start=1):
215
+ leaderboard.append(
216
+ {
217
+ "rank": index,
218
+ "model_id": row["model_id"],
219
+ "run_id": row["run_id"],
220
+ "total_judged": row["total_judged"],
221
+ "total_failures": row["total_failures"],
222
+ "general_score_pct": format_pct(row["general_score"]),
223
+ "highlevel_score_pct": format_pct(row["highlevel_score"]),
224
+ "safety_score_pct": format_pct(row["safety_score"]),
225
+ "basic_score_pct": format_pct(row["basic_score"]),
226
+ "stage_score_pct": format_pct(row["stage_score"]),
227
+ "notes": row["notes"],
228
+ }
229
+ )
230
+ return leaderboard
231
+
232
+
233
+ def markdown_table(rows: list[dict[str, Any]], columns: list[tuple[str, str]]) -> str:
234
+ header = "| " + " | ".join(label for _, label in columns) + " |"
235
+ divider = "| " + " | ".join("---" for _ in columns) + " |"
236
+ body = []
237
+ for row in rows:
238
+ body.append("| " + " | ".join(str(row.get(key, "")) for key, _ in columns) + " |")
239
+ return "\n".join([header, divider, *body])
240
+
241
+
242
+ def write_report(rows: list[dict[str, Any]]) -> None:
243
+ overall = make_overall_table(rows)
244
+ full_scope = make_scope_leaderboard(rows, "full_scope")
245
+ relay_scope = make_scope_leaderboard(rows, "relay_no_safety")
246
+ report_path = OUTPUT_DIR / "README.md"
247
+
248
+ lines = [
249
+ "# Stage Report 2026-05-25",
250
+ "",
251
+ "## Scope",
252
+ "",
253
+ "- This stage report includes only currently usable summaries.",
254
+ "- `gpt-5.4` is a full-scope result.",
255
+ "- `deepseek-v3.2` is a `no-safety` relay result and should only be compared within the same scope.",
256
+ "- Dirty, incomplete, expensive, or incompatible runs such as `gemini-3-flash-preview`, `kimi-k2.6`, `deepseek-r1-250528`, `doubao-seed-2-0-pro-260215`, and `gpt-5.2-pro` are excluded from the leaderboard.",
257
+ "",
258
+ "## Included Runs",
259
+ "",
260
+ markdown_table(
261
+ overall,
262
+ [
263
+ ("model_id", "Model"),
264
+ ("scope", "Scope"),
265
+ ("total_judged", "Judged"),
266
+ ("total_failures", "Failures"),
267
+ ("general_score_pct", "General %"),
268
+ ("highlevel_score_pct", "Highlevel %"),
269
+ ("safety_score_pct", "Safety %"),
270
+ ("basic_score_pct", "Basic %"),
271
+ ("stage_score_pct", "Stage %"),
272
+ ],
273
+ ),
274
+ "",
275
+ "## Leaderboard: Full Scope",
276
+ "",
277
+ markdown_table(
278
+ full_scope,
279
+ [
280
+ ("rank", "Rank"),
281
+ ("model_id", "Model"),
282
+ ("stage_score_pct", "Stage %"),
283
+ ("general_score_pct", "General %"),
284
+ ("safety_score_pct", "Safety %"),
285
+ ("highlevel_score_pct", "Highlevel %"),
286
+ ("basic_score_pct", "Basic %"),
287
+ ],
288
+ ),
289
+ "",
290
+ "## Leaderboard: Relay No-Safety Scope",
291
+ "",
292
+ markdown_table(
293
+ relay_scope,
294
+ [
295
+ ("rank", "Rank"),
296
+ ("model_id", "Model"),
297
+ ("stage_score_pct", "Stage %"),
298
+ ("general_score_pct", "General %"),
299
+ ("highlevel_score_pct", "Highlevel %"),
300
+ ("basic_score_pct", "Basic %"),
301
+ ("total_failures", "Failures"),
302
+ ],
303
+ ),
304
+ "",
305
+ "## Notes",
306
+ "",
307
+ "- `Stage %` is a normalized stage score built from currently available module-level metrics.",
308
+ "- `Basic %` normalizes basic-education tasks onto a 0-100 scale before aggregation.",
309
+ "- Cross-scope comparison is intentionally avoided: full-scope and no-safety relay runs have different included modules.",
310
+ ]
311
+ report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
312
+
313
+
314
+ def main() -> None:
315
+ OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
316
+ rows = build_rows()
317
+
318
+ overall_rows = make_overall_table(rows)
319
+ full_scope_rows = make_scope_leaderboard(rows, "full_scope")
320
+ relay_scope_rows = make_scope_leaderboard(rows, "relay_no_safety")
321
+
322
+ write_csv(
323
+ OUTPUT_DIR / "summary_table.csv",
324
+ overall_rows,
325
+ [
326
+ "model_id",
327
+ "scope",
328
+ "run_id",
329
+ "total_judged",
330
+ "total_failures",
331
+ "general_score_pct",
332
+ "highlevel_score_pct",
333
+ "safety_score_pct",
334
+ "basic_score_pct",
335
+ "stage_score_pct",
336
+ "included_modules",
337
+ "excluded_modules",
338
+ "notes",
339
+ ],
340
+ )
341
+ write_csv(
342
+ OUTPUT_DIR / "leaderboard_full_scope.csv",
343
+ full_scope_rows,
344
+ [
345
+ "rank",
346
+ "model_id",
347
+ "run_id",
348
+ "total_judged",
349
+ "total_failures",
350
+ "general_score_pct",
351
+ "highlevel_score_pct",
352
+ "safety_score_pct",
353
+ "basic_score_pct",
354
+ "stage_score_pct",
355
+ "notes",
356
+ ],
357
+ )
358
+ write_csv(
359
+ OUTPUT_DIR / "leaderboard_relay_no_safety.csv",
360
+ relay_scope_rows,
361
+ [
362
+ "rank",
363
+ "model_id",
364
+ "run_id",
365
+ "total_judged",
366
+ "total_failures",
367
+ "general_score_pct",
368
+ "highlevel_score_pct",
369
+ "basic_score_pct",
370
+ "stage_score_pct",
371
+ "notes",
372
+ ],
373
+ )
374
+
375
+ manifest = {
376
+ "generated_at": "2026-05-25",
377
+ "result_id": "stage-report-20260525",
378
+ "included_runs": [
379
+ {
380
+ "model_id": row["model_id"],
381
+ "run_id": row["run_id"],
382
+ "scope": row["scope"],
383
+ "summary_path": row["summary_path"],
384
+ "notes": row["notes"],
385
+ }
386
+ for row in rows
387
+ ],
388
+ "excluded_runs": [
389
+ {
390
+ "model_id": "kimi-k2.6",
391
+ "reason": "dirty runs; missing raw/judged artifacts",
392
+ },
393
+ {
394
+ "model_id": "deepseek-r1-250528",
395
+ "reason": "dirty runs; timeouts and broken judge endpoint during probe",
396
+ },
397
+ {
398
+ "model_id": "doubao-seed-2-0-pro-260215",
399
+ "reason": "run interrupted by checkpoint write failure",
400
+ },
401
+ {
402
+ "model_id": "gemini-3-flash-preview",
403
+ "reason": "excluded from official scope because the relay repeatedly returns empty content after consuming reasoning tokens",
404
+ },
405
+ {
406
+ "model_id": "gpt-5.2-pro",
407
+ "reason": "expensive GPT pro model intentionally excluded from the current campaign",
408
+ },
409
+ ],
410
+ "generated_files": [
411
+ "README.md",
412
+ "summary_table.csv",
413
+ "leaderboard_full_scope.csv",
414
+ "leaderboard_relay_no_safety.csv",
415
+ "manifest.json",
416
+ ],
417
+ }
418
+ (OUTPUT_DIR / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
419
+ write_report(rows)
420
+
421
+
422
+ if __name__ == "__main__":
423
+ main()
scripts/repair_satas_reference_and_rejudge.py ADDED
@@ -0,0 +1,293 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import asyncio
5
+ import json
6
+ import shutil
7
+ import sys
8
+ from datetime import datetime
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import pandas as pd
13
+
14
+ ROOT = Path(__file__).resolve().parents[1]
15
+ SRC = ROOT / "src"
16
+ if str(SRC) not in sys.path:
17
+ sys.path.insert(0, str(SRC))
18
+
19
+ from elbench.config import load_project_config
20
+ from elbench.judges.judge_teaching_harm import TeachingHarmJudge
21
+ from elbench.loaders import LoaderFactory
22
+ from elbench.persistence import OutputPaths
23
+ from elbench.registry import FileRegistry
24
+ from elbench.schemas.evaluation import ModelResponse, Sample
25
+ from elbench.summary import build_summary
26
+
27
+
28
+ MODELS_AND_RUNS: dict[str, str] = {
29
+ "claude-opus-4-8": "campaign-official-claude-opus-4-8",
30
+ "deepseek-v4-pro": "campaign-official-deepseek-v4-pro",
31
+ "gemini-3.5-flash": "campaign-official-gemini-3.5-flash-aime65536-highlevel-llmjudge",
32
+ "deepseek-v4-flash": "campaign-official-deepseek-v4-flash",
33
+ "doubao-seed-2.0-pro": "campaign-official-doubao-seed-2.0-pro",
34
+ "gpt-5.4": "campaign-official-gpt-5.4-aime65536-highlevel-llmjudge-safety-strict-llmjudge",
35
+ "safe-innospark": "campaign-official-safe-innospark-aime65536-highlevel-llmjudge",
36
+ "innospark-235b": (
37
+ "campaign-official-innospark-235b-aime65536-highlevel-llmjudge-"
38
+ "safety-fill-targeted-safety-strict-llmjudge"
39
+ ),
40
+ "glm-5.1": "campaign-official-glm-5.1",
41
+ }
42
+
43
+ REQUIRED_COLUMNS = ["ID", "Question_Chinese", "Question_English", "Answer"]
44
+
45
+
46
+ def parse_args() -> argparse.Namespace:
47
+ parser = argparse.ArgumentParser(
48
+ description=(
49
+ "Replace the ELBench SATAs sampled file with corrected references and "
50
+ "rejudge existing SATAs model outputs."
51
+ )
52
+ )
53
+ parser.add_argument("--config-root", default="configs")
54
+ parser.add_argument("--corrected-path", type=Path, required=True)
55
+ parser.add_argument("--dry-run", action="store_true")
56
+ return parser.parse_args()
57
+
58
+
59
+ def main() -> None:
60
+ args = parse_args()
61
+ config = load_project_config(args.config_root)
62
+ current_path = _resolve_current_satas_path(config)
63
+ corrected_df = _read_first_sheet(args.corrected_path)
64
+ current_df = _read_first_sheet(current_path)
65
+ sampled_df = _build_corrected_sample(current_df=current_df, corrected_df=corrected_df)
66
+
67
+ if args.dry_run:
68
+ print(
69
+ json.dumps(
70
+ {
71
+ "current_path": str(current_path),
72
+ "corrected_path": str(args.corrected_path),
73
+ "current_rows": len(current_df),
74
+ "corrected_rows": len(corrected_df),
75
+ "sampled_rows": len(sampled_df),
76
+ "changed_answers": int(
77
+ (
78
+ current_df["Answer"].fillna("").astype(str).to_numpy()
79
+ != sampled_df["Answer"].fillna("").astype(str).to_numpy()
80
+ ).sum()
81
+ ),
82
+ },
83
+ ensure_ascii=False,
84
+ indent=2,
85
+ )
86
+ )
87
+ return
88
+
89
+ _replace_satas_file(current_path=current_path, sampled_df=sampled_df)
90
+ sample_by_id = _load_corrected_satas_samples(config)
91
+ results = {}
92
+ for model_id, run_id in MODELS_AND_RUNS.items():
93
+ results[model_id] = asyncio.run(
94
+ _rejudge_model_satas(
95
+ output_root=config.app.output_root,
96
+ model_id=model_id,
97
+ run_id=run_id,
98
+ sample_by_id=sample_by_id,
99
+ )
100
+ )
101
+
102
+ manifest = {
103
+ "generated_at": datetime.now().isoformat(),
104
+ "current_path": str(current_path),
105
+ "corrected_path": str(args.corrected_path),
106
+ "sampled_rows": len(sampled_df),
107
+ "changed_answers": int(
108
+ (
109
+ current_df["Answer"].fillna("").astype(str).to_numpy()
110
+ != sampled_df["Answer"].fillna("").astype(str).to_numpy()
111
+ ).sum()
112
+ ),
113
+ "models": results,
114
+ }
115
+ audit_dir = ROOT / "results" / "audit-judge-integrity"
116
+ audit_dir.mkdir(parents=True, exist_ok=True)
117
+ manifest_path = audit_dir / "satas_reference_rejudge_manifest.json"
118
+ manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
119
+ print(json.dumps(manifest, ensure_ascii=False, indent=2))
120
+
121
+
122
+ def _resolve_current_satas_path(config) -> Path:
123
+ registry = FileRegistry(config)
124
+ items = registry.resolve(source_files={"SATAs.xlsx"})
125
+ if len(items) != 1:
126
+ raise RuntimeError(f"Expected one SATAs registry item, got {len(items)}")
127
+ return items[0].path
128
+
129
+
130
+ def _read_first_sheet(path: Path) -> pd.DataFrame:
131
+ if not path.exists():
132
+ raise FileNotFoundError(path)
133
+ df = pd.read_excel(path, sheet_name=0)
134
+ missing = [column for column in REQUIRED_COLUMNS if column not in df.columns]
135
+ if missing:
136
+ raise ValueError(f"{path} is missing columns: {missing}")
137
+ df = df.loc[:, REQUIRED_COLUMNS].copy()
138
+ if df[REQUIRED_COLUMNS].isna().any(axis=1).any():
139
+ bad_count = int(df[REQUIRED_COLUMNS].isna().any(axis=1).sum())
140
+ raise ValueError(f"{path} has {bad_count} rows with empty required fields")
141
+ if df["ID"].duplicated().any():
142
+ duplicate_ids = df.loc[df["ID"].duplicated(keep=False), "ID"].astype(str).tolist()
143
+ raise ValueError(f"{path} has duplicate IDs: {duplicate_ids[:10]}")
144
+ return df
145
+
146
+
147
+ def _build_corrected_sample(*, current_df: pd.DataFrame, corrected_df: pd.DataFrame) -> pd.DataFrame:
148
+ current_ids = current_df["ID"].astype(str).tolist()
149
+ corrected_by_id = corrected_df.assign(ID=corrected_df["ID"].astype(str)).set_index("ID", drop=False)
150
+ missing = [sample_id for sample_id in current_ids if sample_id not in corrected_by_id.index]
151
+ if missing:
152
+ raise ValueError(f"Corrected SATAs file is missing sampled IDs: {missing[:10]}")
153
+ sampled_df = corrected_by_id.loc[current_ids, REQUIRED_COLUMNS].reset_index(drop=True)
154
+ for column in ["Question_Chinese", "Question_English"]:
155
+ left = current_df[column].fillna("").astype(str).reset_index(drop=True)
156
+ right = sampled_df[column].fillna("").astype(str).reset_index(drop=True)
157
+ if not left.equals(right):
158
+ changed_ids = [
159
+ current_ids[index]
160
+ for index, (left_value, right_value) in enumerate(zip(left, right))
161
+ if left_value != right_value
162
+ ]
163
+ raise ValueError(f"Corrected SATAs changed {column} for sampled IDs: {changed_ids[:10]}")
164
+ return sampled_df
165
+
166
+
167
+ def _replace_satas_file(*, current_path: Path, sampled_df: pd.DataFrame) -> None:
168
+ backup_dir = ROOT / "results" / "audit-judge-integrity" / "satas_backups"
169
+ backup_dir.mkdir(parents=True, exist_ok=True)
170
+ backup_path = backup_dir / f"{current_path.stem}.before_reference_fix.xlsx"
171
+ if not backup_path.exists():
172
+ shutil.copy2(current_path, backup_path)
173
+ sampled_df.to_excel(current_path, index=False)
174
+
175
+
176
+ def _load_corrected_satas_samples(config) -> dict[str, Sample]:
177
+ registry = FileRegistry(config)
178
+ items = registry.resolve(source_files={"SATAs.xlsx"})
179
+ if len(items) != 1:
180
+ raise RuntimeError(f"Expected one SATAs registry item, got {len(items)}")
181
+ loader = LoaderFactory.create(items[0].entry.loader_name)
182
+ samples = list(loader.iter_samples(items[0]))
183
+ if len(samples) != 150:
184
+ raise RuntimeError(f"Expected 150 sampled SATAs records, got {len(samples)}")
185
+ return {sample.sample_id: sample for sample in samples}
186
+
187
+
188
+ async def _rejudge_model_satas(
189
+ *,
190
+ output_root: Path,
191
+ model_id: str,
192
+ run_id: str,
193
+ sample_by_id: dict[str, Sample],
194
+ ) -> dict[str, Any]:
195
+ paths = OutputPaths.build(output_root, run_id, model_id)
196
+ if not paths.judged_path.exists():
197
+ raise FileNotFoundError(paths.judged_path)
198
+ if not paths.raw_path.exists():
199
+ raise FileNotFoundError(paths.raw_path)
200
+
201
+ judged_before = _load_jsonl(paths.judged_path)
202
+ raw_before = _load_jsonl(paths.raw_path)
203
+ judged_after, judged_stats = await _rewrite_records_with_new_satas_judgment(
204
+ records=judged_before,
205
+ sample_by_id=sample_by_id,
206
+ update_judgment=True,
207
+ )
208
+ raw_after, raw_stats = await _rewrite_records_with_new_satas_judgment(
209
+ records=raw_before,
210
+ sample_by_id=sample_by_id,
211
+ update_judgment=False,
212
+ )
213
+ if judged_stats["matched"] != 150:
214
+ raise RuntimeError(f"{model_id} judged SATAs count is {judged_stats['matched']}, expected 150")
215
+ if raw_stats["matched"] != 150:
216
+ raise RuntimeError(f"{model_id} raw SATAs count is {raw_stats['matched']}, expected 150")
217
+
218
+ _write_jsonl(paths.judged_path, judged_after)
219
+ _write_jsonl(paths.raw_path, raw_after)
220
+ summary = build_summary(paths.judged_path, paths.failures_path)
221
+ paths.summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
222
+ return {
223
+ "run_id": run_id,
224
+ "judged_total": len(judged_after),
225
+ "raw_total": len(raw_after),
226
+ "satas_judged": judged_stats,
227
+ "satas_raw": raw_stats,
228
+ "summary_total_failures": summary.get("total_failures"),
229
+ }
230
+
231
+
232
+ async def _rewrite_records_with_new_satas_judgment(
233
+ *,
234
+ records: list[dict[str, Any]],
235
+ sample_by_id: dict[str, Sample],
236
+ update_judgment: bool,
237
+ ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
238
+ judge = TeachingHarmJudge()
239
+ changed_judgments = 0
240
+ matched = 0
241
+ pass_count = 0
242
+ after: list[dict[str, Any]] = []
243
+ for record in records:
244
+ if record.get("task") != "teaching_harm" and record.get("source_file") != "SATAs.xlsx":
245
+ after.append(record)
246
+ continue
247
+ sample = sample_by_id.get(str(record.get("sample_id") or ""))
248
+ if sample is None:
249
+ raise RuntimeError(f"Cannot map SATAs record sample_id={record.get('sample_id')!r}")
250
+ matched += 1
251
+ updated = dict(record)
252
+ updated["source_path"] = sample.source_path
253
+ updated["prompt"] = sample.prompt
254
+ updated["reference"] = sample.reference
255
+ metadata = dict(updated.get("metadata") or {})
256
+ metadata.update(sample.metadata)
257
+ updated["metadata"] = metadata
258
+ if update_judgment:
259
+ previous = (updated.get("judge_result"), updated.get("score"), updated.get("judge_metadata"))
260
+ response = ModelResponse(
261
+ text=updated.get("model_response"),
262
+ raw_payload=updated.get("raw_response") if isinstance(updated.get("raw_response"), dict) else {},
263
+ latency_ms=updated.get("latency_ms"),
264
+ retry_count=int(updated.get("retry_count") or 0),
265
+ )
266
+ judgment = await judge.judge(sample, response)
267
+ updated["judge_result"] = judgment.judge_result
268
+ updated["score"] = judgment.score
269
+ updated["judge_reason"] = judgment.judge_reason
270
+ updated["judge_metadata"] = judgment.judge_metadata
271
+ current = (updated.get("judge_result"), updated.get("score"), updated.get("judge_metadata"))
272
+ if current != previous:
273
+ changed_judgments += 1
274
+ if updated.get("judge_result") == "pass":
275
+ pass_count += 1
276
+ after.append(updated)
277
+ return after, {"matched": matched, "changed_judgments": changed_judgments, "pass_count": pass_count}
278
+
279
+
280
+ def _load_jsonl(path: Path) -> list[dict[str, Any]]:
281
+ return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
282
+
283
+
284
+ def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
285
+ temp_path = path.with_suffix(path.suffix + ".tmp")
286
+ with temp_path.open("w", encoding="utf-8") as handle:
287
+ for record in records:
288
+ handle.write(json.dumps(record, ensure_ascii=False, default=str) + "\n")
289
+ temp_path.replace(path)
290
+
291
+
292
+ if __name__ == "__main__":
293
+ main()
scripts/run_benchmark.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ import sys
3
+
4
+ ROOT = Path(__file__).resolve().parents[1]
5
+ SRC = ROOT / "src"
6
+ if str(SRC) not in sys.path:
7
+ sys.path.insert(0, str(SRC))
8
+
9
+ from elbench.cli import main
10
+
11
+
12
+ if __name__ == "__main__":
13
+ main()
14
+
src/elbench/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Education benchmark framework."""
src/elbench/basic_education_runtime/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ """Internal runtime for ELBench basic education scenarios."""
2
+
src/elbench/basic_education_runtime/agent.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain.chat_models.base import BaseChatModel
2
+ from langgraph.prebuilt.chat_agent_executor import AgentState
3
+ from langgraph.graph import StateGraph, START, END
4
+ from langgraph.graph.state import CompiledStateGraph
5
+ from langchain_core.messages import AIMessage, HumanMessage
6
+
7
+ from typing import Any, Dict, List, Callable, Optional, Tuple, Awaitable
8
+ from tenacity import retry, stop_after_attempt, wait_fixed
9
+
10
+ from elbench.basic_education_runtime.entity import AgentConfig
11
+ from elbench.basic_education_runtime.utils import content_to_text, replace_prompt, remove_think
12
+ from elbench.basic_education_runtime.config import CONFIG
13
+
14
+ import asyncio
15
+ import copy
16
+
17
+
18
+ def _init_agent_from_dict(
19
+ ac: AgentConfig,
20
+ model_map: Dict[str, BaseChatModel],
21
+ agent_name: str,
22
+ dynamic_prompt_map: Optional[Dict[str, str]] = None,
23
+ ) -> Callable[..., Awaitable[Dict[str, List[Any]]]]:
24
+ if dynamic_prompt_map is not None:
25
+ ac_prompt = replace_prompt(ac.prompt, dynamic_prompt_map)
26
+ else:
27
+ ac_prompt = copy.deepcopy(ac.prompt) # fix shallow copy瀵艰嚧鐨勬暟鎹敊璇?
28
+ m = model_map[ac.model]
29
+
30
+ @retry(
31
+ stop=stop_after_attempt(CONFIG.globals.retry.attempt),
32
+ wait=wait_fixed(CONFIG.globals.retry.interval),
33
+ )
34
+ async def chatbot(state: AgentState) -> Dict[str, List[Any]]:
35
+ if state["messages"] == []:
36
+ n_m = ac_prompt
37
+ else:
38
+ n_m = []
39
+ for item in state["messages"]:
40
+ content = content_to_text(remove_think(item.content))
41
+ if not content.strip():
42
+ continue
43
+ if item.name == agent_name:
44
+ item = AIMessage(content=content, type="ai") # type: ignore
45
+ else:
46
+ item = HumanMessage(content=content, type="human") # type: ignore
47
+ n_m.append(item)
48
+ if len(n_m) > ac.memory.keep_turns * 2 + 1:
49
+ n_m = n_m[-ac.memory.keep_turns * 2 - 1 :]
50
+ n_m = ac_prompt + n_m
51
+ r = await asyncio.wait_for( # type: ignore
52
+ m.ainvoke(n_m),
53
+ timeout=CONFIG.globals.model_call_timeout_seconds,
54
+ )
55
+ r.name = agent_name
56
+ return {"messages": [r]}
57
+
58
+ return chatbot
59
+
60
+
61
+ def init_agent_map_from_dict(
62
+ model_map: Dict[str, BaseChatModel],
63
+ dynamic_prompt_map: Optional[Dict[str, str]] = None,
64
+ ) -> Tuple[Dict[str, Tuple[CompiledStateGraph, AgentConfig]], Optional[Dict[str, str]]]:
65
+ result = {}
66
+ acs = CONFIG.agents
67
+ for k, v in acs.items():
68
+ ac = v
69
+ if ac.memory.enable:
70
+ # conn = sqlite3.connect(f"{k}_checkpoint.sqlite", check_same_thread=False)
71
+ # memory = SqliteSaver(conn)
72
+ memory = True
73
+ else:
74
+ memory = None
75
+ model = _init_agent_from_dict(ac, model_map, k, dynamic_prompt_map)
76
+ graph = StateGraph(AgentState)
77
+ graph.add_node("agent", model)
78
+ graph.add_edge(START, "agent")
79
+ graph.add_edge("agent", END)
80
+ result[k] = (graph.compile(checkpointer=memory), ac)
81
+ return result, dynamic_prompt_map
82
+
83
+
84
+ init_agent_map = init_agent_map_from_dict
85
+
src/elbench/basic_education_runtime/assets/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Assets used by the internal basic education runtime."""
src/elbench/basic_education_runtime/assets/fonts/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Font assets for the internal basic education runtime."""
src/elbench/basic_education_runtime/assets/fonts/sarasa-mono-sc-regular.ttf ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22d82510f7ac3b342668055e1c6e935b80962e86abd0f2e3c1ad9919a2419b21
3
+ size 22515080
src/elbench/basic_education_runtime/cli/draw/__init__.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+ from elbench.basic_education_runtime.langchain_compat import set_debug
3
+
4
+
5
+ @click.command(help="Draw Agent workflow.")
6
+ @click.option(
7
+ "--config", type=click.Path(exists=True), help="Path to the configuration file"
8
+ )
9
+ @click.option("--debug", is_flag=True, help="Enable debug mode")
10
+ def draw(config, debug=False):
11
+ set_debug(debug)
12
+ from elbench.basic_education_runtime.config import load_conf
13
+ from pathlib import Path
14
+
15
+ config = Path(config)
16
+ load_conf(config)
17
+
18
+ from elbench.basic_education_runtime.config import CONFIG
19
+ from elbench.basic_education_runtime.model import init_model_map
20
+ from elbench.basic_education_runtime.agent import init_agent_map
21
+ from elbench.basic_education_runtime.directions import apply_agent_direction
22
+
23
+ # from langchain_core.runnables.graph import CurveStyle, NodeStyles, MermaidDrawMethod
24
+
25
+ import asyncio
26
+
27
+ models = init_model_map()
28
+ task = CONFIG.tasks.variables[0]
29
+ agents, _ = init_agent_map(models, task)
30
+ agent, _ = asyncio.run(apply_agent_direction(agents, task=task))
31
+ png = agent.get_graph().draw_mermaid_png()
32
+ with open(f"{config.stem}.png", "wb") as wb:
33
+ wb.write(png)
34
+ # print(png)
35
+
src/elbench/basic_education_runtime/cli/eval/__init__.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+
3
+ from pathlib import Path
4
+ from typing import Dict, Any
5
+
6
+ from elbench.basic_education_runtime.langchain_compat import set_debug
7
+ from tenacity import RetryError
8
+
9
+
10
+ @click.command(help="Evaluate the performance of a model on a dataset")
11
+ @click.option(
12
+ "--config",
13
+ type=click.Path(exists=True),
14
+ required=True,
15
+ help="Path to the configuration file",
16
+ )
17
+ @click.option("--debug", default=False, help="Debug Mode", is_flag=True)
18
+ @click.option("--avg/--no-avg", default=True, help="Calculate the average score")
19
+ def eval(config: Path, debug: bool, avg: bool):
20
+ set_debug(debug)
21
+ from elbench.basic_education_runtime.config import load_conf
22
+
23
+ load_conf(config)
24
+ eval_logic(avg)
25
+
26
+
27
+ def eval_logic(avg: bool):
28
+ from elbench.basic_education_runtime.config import CONFIG
29
+
30
+ input_dir = CONFIG.globals.memory.path
31
+ import asyncio
32
+ from elbench.basic_education_runtime.evaluation import evaluate
33
+ from elbench.basic_education_runtime.model import init_chat_model_from_dict
34
+
35
+ from elbench.basic_education_runtime.entity import ExportFormat
36
+ from tqdm.asyncio import tqdm
37
+ import json
38
+
39
+ input_dir = Path(input_dir)
40
+
41
+ eval_path = input_dir / "eval"
42
+ eval_path.mkdir(exist_ok=True)
43
+
44
+ sem = asyncio.Semaphore(CONFIG.globals.concurrency)
45
+
46
+ async def eval_task(model, file: Path) -> Dict[str, Any]:
47
+ async with sem:
48
+ ef = ExportFormat.from_json_file(file)
49
+ try:
50
+ eval = await evaluate(model, ef)
51
+ with open(eval_path / file.name, "w", encoding="utf8") as f:
52
+ json.dump(eval, f, ensure_ascii=False, indent=4)
53
+ return eval
54
+ except RetryError as e:
55
+ print(f"Error evaluating {file}", e.last_attempt.exception())
56
+ return {}
57
+
58
+ async def main():
59
+ assert CONFIG.evaluation
60
+ model = init_chat_model_from_dict(CONFIG.models[CONFIG.evaluation.model])
61
+
62
+ to_eval_files = list(input_dir.glob("*.json"))
63
+ task_ids = [file.stem for file in to_eval_files]
64
+ eval_tasks = []
65
+ for file in to_eval_files:
66
+ eval_tasks.append(eval_task(model, file))
67
+
68
+ evals = await tqdm.gather(*eval_tasks)
69
+
70
+ csv_utf8 = open(
71
+ eval_path / f"{CONFIG.evaluation.name}.csv", "w", encoding="utf-8"
72
+ )
73
+ # csv_gbk = open(eval_path / f"{CONFIG.evaluation.name}-gbk.csv", "w", encoding="gbk")
74
+
75
+ metric_fields = _metric_fields_from_evals(evals)
76
+ if not metric_fields:
77
+ csv_utf8.write("task_id\n")
78
+ csv_utf8.close()
79
+ raise RuntimeError("No successful evaluation outputs were produced.")
80
+
81
+ title = ["task_id"]
82
+ for field in metric_fields:
83
+ title.append(field)
84
+
85
+ if avg:
86
+ title.append("avg")
87
+
88
+ csv_utf8.write(",".join(title) + "\n")
89
+ # csv_gbk.write(",".join(title) + "\n")
90
+
91
+ if avg:
92
+ row = len(task_ids) + 1
93
+ metric_count = len(metric_fields)
94
+ col = metric_count + 1
95
+
96
+ matrix = [[0.0] * col for _ in range(row)]
97
+
98
+ # 缁熻鏁版嵁骞惰绠楁瘡琛屽钩鍧囧€?
99
+ for idx, (task_id, eval) in enumerate(zip(task_ids, evals)):
100
+ contents = [task_id]
101
+ for sub_idx, field in enumerate(metric_fields):
102
+ raw_value = eval.get(field, 0)
103
+ v = float(raw_value)
104
+ matrix[idx][sub_idx] = v
105
+ contents.append(f"{raw_value}")
106
+ sum = 0
107
+ for i in matrix[idx][:-1]:
108
+ sum += i
109
+ # 鏈€鍚庝竴鍒楃殑鏁板瓧 = 姣忓垪鐨勫拰闄や互鍒楁暟-1
110
+ matrix[idx][col - 1] = sum / metric_count
111
+ contents.append(f"{matrix[idx][col - 1]:.2f}")
112
+ csv_utf8.write(",".join(contents) + "\n")
113
+ # csv_gbk.write(",".join(contents) + "\n")
114
+ # 璁$畻姣忓垪鐨勫钩鍧囧€?
115
+ for col_idx in range(col):
116
+ # print(matrix)
117
+ sum = 0
118
+ # 璁$畻姣忎竴鍒楅櫎鍘绘渶鍚庝竴涓厓绱犵殑鍜?
119
+ for row_idx in range(row - 1):
120
+ sum += matrix[row_idx][col_idx]
121
+ matrix[-1][col_idx] = sum / (row - 1)
122
+
123
+ write_str = ["%.2f" % i for i in matrix[-1]]
124
+ write_str.insert(0, "Avg")
125
+ # 鍐欏叆鏈€鍚庝竴琛岀殑骞冲潎鍊?
126
+ csv_utf8.write(",".join(write_str) + "\n")
127
+ # csv_gbk.write(",".join(write_str) + "\n")
128
+ else:
129
+ for task_id, eval in zip(task_ids, evals):
130
+ contents = [task_id]
131
+ for field in metric_fields:
132
+ contents.append(f"{eval.get(field, 0)}")
133
+ csv_utf8.write(",".join(contents) + "\n")
134
+ # csv_gbk.write(",".join(contents) + "\n")
135
+
136
+ csv_utf8.close()
137
+ # csv_gbk.close()
138
+
139
+ asyncio.run(main())
140
+
141
+
142
+ def _metric_fields_from_evals(evals: list[Dict[str, Any]]) -> list[str]:
143
+ for item in evals:
144
+ if item:
145
+ return list(item.keys())
146
+ return []
147
+
src/elbench/basic_education_runtime/cli/export/__init__.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+
3
+ from elbench.basic_education_runtime.cli.export.json_ import json
4
+
5
+ try:
6
+ from elbench.basic_education_runtime.cli.export.label_studio import label_studio
7
+ except Exception: # optional command; do not block core pipeline/export-json flow
8
+ label_studio = None
9
+
10
+
11
+ @click.group(help="Export chat databases")
12
+ def export():
13
+ pass
14
+
15
+
16
+ export.add_command(json)
17
+ if label_studio is not None:
18
+ export.add_command(label_studio)
19
+
src/elbench/basic_education_runtime/cli/export/const/label_studio.py ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Sequence
2
+ from elbench.basic_education_runtime.entity import FormatField
3
+
4
+
5
+ TEMPLATE_template = """<View>
6
+ <Style>
7
+ .container {
8
+ display: flex;
9
+ justify-content: space-between;
10
+ margin: 0 auto;
11
+ padding: 20px;
12
+ background-color: #ffffff;
13
+ border-radius: 5px;
14
+ box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.1), 0 6px 20px 0 rgba(0, 0, 0, 0.1);
15
+ max-width: 800px;
16
+ }
17
+
18
+ .border {
19
+ border-style: solid;
20
+ }
21
+
22
+ .text-block {
23
+ flex: 1;
24
+ margin-right: 20px;
25
+ }
26
+
27
+ .assessment-items-container {
28
+ flex: 1;
29
+ display: flex;
30
+ flex-direction: column;
31
+ gap: 20px;
32
+ }
33
+
34
+ .assessment-item {
35
+ background-color: rgba(44, 62, 80, 0.6);
36
+ padding: 1px;
37
+ border-radius: 5px;
38
+ box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.1), 0 3px 10px 0 rgba(0, 0, 0, 0.1);
39
+ color: #ffffff;
40
+ word-wrap: break-word;
41
+ }
42
+ </Style>
43
+ <View className="container">
44
+ <View className="text-block">
45
+ <Paragraphs name="dialogue" value="$messages" layout="dialogue" nameKey="role" textKey="content" />
46
+ </View>
47
+ <View className="assessment-item-container">
48
+ """
49
+
50
+ TEMPLATE_SUFFIX = """
51
+ </View>
52
+ </View>
53
+ </View>
54
+ """
55
+
56
+
57
+ def generate_labeling(fields: Sequence[FormatField]) -> str:
58
+ compoents: list[str] = []
59
+ for field in fields:
60
+ name = field.field
61
+ template = f' <Header value="{name}" size="8"/><View className="assessment-item">%s</View>\n'
62
+ if field.type == "int":
63
+ if field.max is not None:
64
+ compoents.append(template % f'<Rating name="{name}" maxRating="{field.max}" toName="dialogue" />')
65
+ else:
66
+ compoents.append(template % f'<Rating name="{name}" maxRating="5" toName="dialogue" />')
67
+ elif field.type == "float":
68
+ if field.max is not None:
69
+ compoents.append(template % f'<Number name="{name}" toName="dialogue" max="{field.max}" />')
70
+ else:
71
+ compoents.append(template % f'<Number name="{name}" toName="dialogue" />')
72
+ elif field.type == "str":
73
+ compoents.append(template % f'<TextArea name="{name}" toName="dialogue" />')
74
+ elif field.type == "bool":
75
+ compoents.append(template % f'<Choices name="{name}" toName="dialogue" showInline="true" choice="single-radio"><Choice value="Yes"/><Choice value="No"/></Choices>')
76
+ elif field.type == "dict":
77
+ children = generate_labeling(field.items)
78
+ compoents.append(template % f'<View className="border">{"".join(children)}</View>')
79
+ else:
80
+ raise NotImplementedError
81
+ return "".join(compoents)
82
+
83
+ def generate_label_studio_interface(fields: Sequence[FormatField]) -> str:
84
+ return TEMPLATE_template + generate_labeling(fields) + TEMPLATE_SUFFIX
85
+
86
+
87
+
88
+
89
+
src/elbench/basic_education_runtime/cli/export/exporter/json_.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import sqlite3
2
+ from typing import Tuple, Dict, Any
3
+ from pathlib import Path
4
+ import json
5
+ from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
6
+ from langgraph.checkpoint.base import Checkpoint
7
+
8
+
9
+ def _to_jsonable(value: Any) -> Any:
10
+ return json.loads(json.dumps(value, ensure_ascii=False, default=str))
11
+
12
+
13
+ def _stringify_block_text(value: Any) -> str:
14
+ if value is None:
15
+ return ""
16
+ if isinstance(value, str):
17
+ return value
18
+ if isinstance(value, list):
19
+ parts: list[str] = []
20
+ for item in value:
21
+ if isinstance(item, dict):
22
+ text = item.get("text")
23
+ if text not in (None, ""):
24
+ parts.append(str(text))
25
+ continue
26
+ if item not in (None, ""):
27
+ parts.append(str(item))
28
+ return "\n".join(parts).strip()
29
+ if isinstance(value, dict):
30
+ text = value.get("text")
31
+ if text not in (None, ""):
32
+ return str(text)
33
+ return json.dumps(_to_jsonable(value), ensure_ascii=False)
34
+ return str(value)
35
+
36
+
37
+ def _split_reasoning_and_response(content: Any) -> tuple[str, str]:
38
+ if isinstance(content, str):
39
+ if "</think>" in content:
40
+ content_split = content.split("</think>", 1)
41
+ return content_split[0].strip(), content_split[1].strip()
42
+ return "", content.strip()
43
+
44
+ if isinstance(content, list):
45
+ reasoning_parts: list[str] = []
46
+ response_parts: list[str] = []
47
+ for block in content:
48
+ if isinstance(block, dict):
49
+ block_type = str(block.get("type", "")).strip().lower()
50
+ if block_type == "reasoning":
51
+ reasoning_text = _stringify_block_text(block.get("summary"))
52
+ if not reasoning_text:
53
+ reasoning_text = _stringify_block_text(block.get("text"))
54
+ if reasoning_text:
55
+ reasoning_parts.append(reasoning_text)
56
+ continue
57
+ if block_type in {"text", "output_text"}:
58
+ text = _stringify_block_text(block.get("text"))
59
+ if text:
60
+ response_parts.append(text)
61
+ continue
62
+ fallback = _stringify_block_text(block)
63
+ if fallback:
64
+ response_parts.append(fallback)
65
+ continue
66
+
67
+ fallback = _stringify_block_text(block)
68
+ if fallback:
69
+ response_parts.append(fallback)
70
+ return "\n".join(reasoning_parts).strip(), "\n".join(response_parts).strip()
71
+
72
+ response = _stringify_block_text(content).strip()
73
+ return "", response
74
+
75
+
76
+ async def aexport_json(input_path: Path) -> Tuple[Path, Dict[str, Any]]:
77
+ conn = sqlite3.connect(input_path)
78
+ cursor = conn.cursor()
79
+ sql = "select checkpoint_ns, checkpoint_id, parent_checkpoint_id, checkpoint from checkpoints"
80
+ cursor.execute(sql)
81
+ results = cursor.fetchall()[-1]
82
+ cns, cid, pcid, c = results
83
+ jps = JsonPlusSerializer()
84
+ checkpoint: Checkpoint = jps.loads_typed(("msgpack", c))
85
+ messages = []
86
+ for m in checkpoint.get("channel_values")["messages"]:
87
+ if m.name is None:
88
+ continue
89
+ reasoning, response = _split_reasoning_and_response(getattr(m, "content", None))
90
+ item = {"role": m.name, "content": response, "reasoning": reasoning}
91
+ usage_metadata = getattr(m, "usage_metadata", None)
92
+ response_metadata = getattr(m, "response_metadata", None)
93
+ additional_kwargs = getattr(m, "additional_kwargs", None)
94
+ if usage_metadata not in (None, {}):
95
+ item["usage_metadata"] = _to_jsonable(usage_metadata)
96
+ if response_metadata not in (None, {}):
97
+ item["response_metadata"] = _to_jsonable(response_metadata)
98
+ if additional_kwargs not in (None, {}):
99
+ item["additional_kwargs"] = _to_jsonable(additional_kwargs)
100
+ messages.append(item)
101
+
102
+ sql = "select key, value from task"
103
+ cursor.execute(sql)
104
+ results = cursor.fetchall()
105
+ obj = {"task": {}, "messages": messages}
106
+
107
+ for key, value in results:
108
+ obj["task"][key] = value
109
+
110
+ return input_path, obj
src/elbench/basic_education_runtime/cli/export/exporter/label_studio_.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ from typing import Dict, Any, List
3
+
4
+ from elbench.basic_education_runtime.cli.export.exporter.json_ import aexport_json
5
+
6
+ async def aexport_label_studio(input_path: Path) -> Dict[str, Any]:
7
+ _, json_obj = await aexport_json(input_path)
8
+ msgs = []
9
+ for msg in json_obj["messages"]:
10
+ msgs.append({
11
+ "role": msg["role"],
12
+ "content": msg["content"]
13
+ })
14
+ return {
15
+ "data": {
16
+ "messages": msgs,
17
+ "task_id": input_path.stem
18
+ }
19
+ }
src/elbench/basic_education_runtime/cli/export/json_.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json as jsonmodule
2
+ import asyncio
3
+ from tqdm.asyncio import tqdm
4
+
5
+ import click
6
+
7
+ from pathlib import Path
8
+
9
+ from elbench.basic_education_runtime.langchain_compat import set_debug
10
+
11
+
12
+ def export_json_logic():
13
+ from elbench.basic_education_runtime.config import CONFIG
14
+
15
+ input = CONFIG.globals.memory.path
16
+ output = input
17
+
18
+ dbfiles = []
19
+ files = input.iterdir()
20
+ for file in files:
21
+ if file.suffix == ".db":
22
+ dbfiles.append(file.absolute())
23
+
24
+ from elbench.basic_education_runtime.cli.export.exporter.json_ import aexport_json
25
+
26
+ tasks = []
27
+ for dbfile in dbfiles:
28
+ task = aexport_json(dbfile)
29
+ tasks.append(task)
30
+
31
+ result = asyncio.run(tqdm.gather(*tasks))
32
+ for input_path, obj in result:
33
+ output_path = output / f"{input_path.stem}.json"
34
+ with open(output_path, "w", encoding="utf-8") as f:
35
+ jsonmodule.dump(obj, f, ensure_ascii=False, indent=4)
36
+
37
+
38
+ @click.command(help="Export chat databases to JSON format")
39
+ @click.option(
40
+ "--config", default="config.yaml", help="Directory containing chat databases"
41
+ )
42
+ @click.option("--debug", default=False, help="Debug Mode", is_flag=True)
43
+ def json(config: str, debug: bool):
44
+ set_debug(debug)
45
+ from elbench.basic_education_runtime.config import load_conf
46
+
47
+ path = Path(config)
48
+ load_conf(path)
49
+ export_json_logic()
50
+
src/elbench/basic_education_runtime/cli/export/label_studio.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ from pathlib import Path
3
+ from tqdm.asyncio import tqdm
4
+ from typing import Dict, Any
5
+
6
+ import json as jsonmodule
7
+ import asyncio
8
+ from tqdm.asyncio import tqdm
9
+
10
+ import click
11
+
12
+ from pathlib import Path
13
+
14
+ from elbench.basic_education_runtime.langchain_compat import set_debug
15
+
16
+
17
+ def export_label_studio_logic():
18
+ from elbench.basic_education_runtime.config import CONFIG
19
+
20
+ input = CONFIG.globals.memory.path
21
+ output = input
22
+
23
+ dbfiles = []
24
+ files = input.iterdir()
25
+ for file in files:
26
+ if file.suffix == ".db":
27
+ dbfiles.append(file.absolute())
28
+
29
+ from elbench.basic_education_runtime.cli.export.exporter.label_studio_ import aexport_label_studio
30
+ from elbench.basic_education_runtime.cli.export.const.label_studio import generate_label_studio_interface
31
+
32
+ if CONFIG.evaluation is None:
33
+ raise ValueError("Evaluation瀛楁鏈厤缃?)
34
+
35
+ tasks = []
36
+ for dbfile in dbfiles:
37
+ task = aexport_label_studio(dbfile)
38
+ tasks.append(task)
39
+
40
+ results = asyncio.run(tqdm.gather(*tasks))
41
+ with open(output / f"label_studio.json", "w", encoding="utf-8") as f:
42
+ jsonmodule.dump(results, f, ensure_ascii=False, indent=4)
43
+
44
+ template = generate_label_studio_interface(CONFIG.evaluation.format)
45
+ with open(output / "label_studio.txt", "w", encoding="utf-8") as f:
46
+ f.write(template)
47
+
48
+
49
+
50
+ @click.command(help="Export chat databases to Label Studio Data format")
51
+ @click.option(
52
+ "--config", default="config.yaml", help="Directory containing chat databases"
53
+ )
54
+ @click.option("--debug", default=False, help="Debug Mode", is_flag=True)
55
+ def label_studio(config: str, debug: bool):
56
+ set_debug(debug)
57
+ from elbench.basic_education_runtime.config import load_conf
58
+
59
+ path = Path(config)
60
+ load_conf(path)
61
+ export_label_studio_logic()
src/elbench/basic_education_runtime/cli/generate/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+
3
+ from pathlib import Path
4
+ from elbench.basic_education_runtime.langchain_compat import set_debug
5
+
6
+ # set_debug(True)
7
+
8
+
9
+ @click.command("generate", help="Generate conversions for all tasks")
10
+ @click.option("--config", default="config.yaml", help="Path to the configuration file.")
11
+ @click.option("--debug", default=False, help="Debug Mode", is_flag=True)
12
+ def generate(config: str, debug: bool):
13
+ set_debug(debug)
14
+ from elbench.basic_education_runtime.config import load_conf
15
+
16
+ path = Path(config)
17
+ load_conf(path)
18
+ generate_logic()
19
+
20
+
21
+ def generate_logic():
22
+ from elbench.basic_education_runtime.run import run
23
+ import asyncio
24
+
25
+ asyncio.run(run())
26
+
src/elbench/basic_education_runtime/cli/main.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+
3
+ from elbench.basic_education_runtime.langchain_compat import set_debug
4
+
5
+ from elbench.basic_education_runtime.cli.generate import generate, generate_logic
6
+ from elbench.basic_education_runtime.cli.eval import eval, eval_logic
7
+ from elbench.basic_education_runtime.cli.visualize import visualize
8
+ from elbench.basic_education_runtime.cli.draw import draw
9
+ from elbench.basic_education_runtime.cli.export import export
10
+ from elbench.basic_education_runtime.cli.export.json_ import export_json_logic
11
+
12
+
13
+ @click.command(
14
+ help="Run the pipeline to generate, export JSON files, and evaluate the results."
15
+ )
16
+ @click.option(
17
+ "--config", type=click.Path(exists=True), help="Path to the configuration file"
18
+ )
19
+ @click.option("--debug", is_flag=True, help="Enable debug mode")
20
+ def pipeline(config, debug=False):
21
+ set_debug(debug)
22
+ from elbench.basic_education_runtime.config import load_conf
23
+
24
+ load_conf(config)
25
+ generate_logic()
26
+ export_json_logic()
27
+ eval_logic(avg=True)
28
+
29
+
30
+ @click.group()
31
+ def main():
32
+ pass
33
+
34
+
35
+ main.add_command(generate)
36
+ main.add_command(export)
37
+ main.add_command(eval)
38
+ main.add_command(pipeline)
39
+
40
+ main.add_command(visualize)
41
+
42
+ main.add_command(draw)
43
+
44
+
45
+ if __name__ == "__main__":
46
+ main()
src/elbench/basic_education_runtime/cli/visualize/__init__.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import click
2
+
3
+ from pathlib import Path
4
+
5
+
6
+ @click.command(help="Visualize the results in all CSV file in the specified directory.")
7
+ @click.argument(
8
+ "input_dir",
9
+ type=click.Path(exists=True),
10
+ )
11
+ @click.option(
12
+ "--x-rotation",
13
+ type=int,
14
+ default=30,
15
+ )
16
+ def visualize(input_dir: str, x_rotation: int):
17
+ visualize_logic(input_dir, x_rotation)
18
+
19
+
20
+ def visualize_logic(input_dir: str, x_rotation: int):
21
+ color_palette = [
22
+ "#1f77b4",
23
+ "#aec7e8",
24
+ "#ff7f0e",
25
+ "#ffbb78",
26
+ "#2ca02c",
27
+ "#98df8a",
28
+ "#d62728",
29
+ "#ff9896",
30
+ "#9467bd",
31
+ "#c5b0d5",
32
+ "#8c564b",
33
+ "#c49c94",
34
+ "#e377c2",
35
+ "#f7b6d2",
36
+ "#7f7f7f",
37
+ "#c7c7c7",
38
+ "#bcbd22",
39
+ "#dbdb8d",
40
+ "#17becf",
41
+ "#9edae5",
42
+ ]
43
+ import pandas as pd
44
+ import matplotlib.pyplot as plt
45
+ import numpy as np
46
+ from matplotlib import font_manager
47
+
48
+ from importlib.resources import files
49
+
50
+ font_path = files("elbench.basic_education_runtime.assets.fonts").joinpath("sarasa-mono-sc-regular.ttf")
51
+ font_path = str(font_path)
52
+
53
+ font_manager.fontManager.addfont(font_path)
54
+ plt.rcParams["font.sans-serif"] = "Sarasa Mono SC"
55
+ plt.rcParams["axes.unicode_minus"] = False # 瑙e喅璐熷彿鏄剧ず闂
56
+
57
+ input_path = Path(input_dir)
58
+ csvs = input_path.rglob("*.csv")
59
+
60
+ task_name = ""
61
+ keys = []
62
+ models = []
63
+ values = {}
64
+
65
+ for csv in csvs:
66
+ stem_split = csv.stem.rsplit("_", 1)
67
+ if task_name == "":
68
+ task_name = stem_split[0]
69
+ elif task_name != stem_split[0]:
70
+ raise ValueError(
71
+ f"Multiple task names found in CSV files. {task_name} and {stem_split[0]} are different."
72
+ )
73
+
74
+ model = stem_split[1]
75
+ models.append(model)
76
+
77
+ data = pd.read_csv(csv)
78
+ data = data.drop(columns=["task_id", "avg"])
79
+ data = data.iloc[-1].to_dict()
80
+
81
+ if not keys:
82
+ keys = list(data.keys())
83
+ for k in keys:
84
+ values[k] = []
85
+ elif keys != list(data.keys()):
86
+ raise ValueError(
87
+ f"Data keys do not match across CSV files. [{','.join(keys)}] and [{','.join(data.keys())}] are different."
88
+ )
89
+
90
+ for k in keys:
91
+ values[k].append(data[k])
92
+
93
+ # 鏋勫缓 DataFrame
94
+ df_dict = {"": models}
95
+ for k in keys:
96
+ df_dict[k] = values[k]
97
+
98
+ df = pd.DataFrame(df_dict)
99
+
100
+ ncol = min(len(keys), 5)
101
+
102
+ # ==== 鉁?鑷€傚簲鐢诲竷瀹藉害 ====
103
+ fig_width = max(8, len(df) * 0.8, ncol * 2) # 姣忎釜妯″瀷 0.8 鑻卞锛屾渶灏忓搴︿负 8
104
+ fig, ax = plt.subplots(figsize=(fig_width, 6))
105
+
106
+ df.set_index("").plot(kind="bar", stacked=True, ax=ax, color=color_palette)
107
+ ax.set_xticklabels(df[""], rotation=x_rotation)
108
+ ax.set_title(f"{task_name}")
109
+
110
+ # 鉁?璁剧疆鍥句緥浣嶇疆鍒板浘琛ㄤ笅鏂癸紝鎵撴暎涓哄鍒?
111
+ ax.legend(
112
+ loc="lower center",
113
+ bbox_to_anchor=(0.5, 1.1),
114
+ ncol=ncol,
115
+ frameon=False,
116
+ )
117
+
118
+ plt.tight_layout()
119
+ plt.savefig(input_path / f"stack_{task_name}.png", dpi=300)
120
+
121
+ # ==== 鉁?闆疯揪鍥?====
122
+ # 鍑嗗鏁版嵁
123
+ num_vars = len(keys)
124
+ angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()
125
+ angles += angles[:1] # 闂悎鍥惧舰
126
+
127
+ # 璁$畻鎵€鏈夋暟鍊肩殑鏈€灏忓€?
128
+ all_scores = [values[k] for k in keys]
129
+ min_value = min([min(score_list) for score_list in all_scores])
130
+
131
+ # 璁剧疆闆疯揪鍥剧殑鏈€灏忓€煎師鐐?
132
+ min_value -= 1 # 鏈€灏忓€煎噺鍘?1
133
+
134
+ # 缁樺埗闆疯揪鍥?
135
+ fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(polar=True))
136
+
137
+ for idx, model in enumerate(models):
138
+ scores = [values[k][idx] for k in keys]
139
+ scores += scores[:1] # 闂悎鍥惧舰
140
+ ax.plot(
141
+ angles,
142
+ scores,
143
+ label=model,
144
+ linewidth=2,
145
+ color=color_palette[idx % len(color_palette)],
146
+ )
147
+ ax.fill(angles, scores, alpha=0.1)
148
+
149
+ # 璋冩暣闆疯揪鍥剧殑鍗婂緞鑼冨洿锛岀‘淇濅粠 (min_value - 1) 寮€濮?
150
+ ax.set_ylim(min_value, max([max(score_list) for score_list in all_scores]) + 1)
151
+
152
+ # 璁剧疆鏍囩鍜屾牱寮?
153
+ ax.set_thetagrids(np.degrees(angles[:-1]), keys) # type: ignore
154
+ ax.set_title(f"{task_name}", size=16)
155
+ ax.legend(loc="upper right", bbox_to_anchor=(1.3, 1.1))
156
+ plt.tight_layout()
157
+ plt.savefig(input_path / f"radar_{task_name}.png", dpi=300)
158
+
src/elbench/basic_education_runtime/config.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from elbench.basic_education_runtime.entity import BasicEducationRuntimeConfig
2
+ from elbench.basic_education_runtime.utils import extract
3
+ from pathlib import Path
4
+ from typing import Dict, Any
5
+
6
+ import yaml
7
+
8
+ CONFIG: BasicEducationRuntimeConfig
9
+
10
+
11
+ def load_conf(path: Path):
12
+ if isinstance(path, str):
13
+ path = Path(path)
14
+ if not path.exists():
15
+ return
16
+ global CONFIG
17
+ data: Dict[str, Dict[str, Any]] = {}
18
+ try:
19
+ with open(path, "r", encoding="utf8") as f:
20
+ t = f.read()
21
+ for d in yaml.safe_load_all(t):
22
+ data = d
23
+ # 缂栫爜閿欒
24
+ except UnicodeDecodeError:
25
+ with open(path, "r", encoding="gbk") as f:
26
+ t = f.read()
27
+ for d in yaml.safe_load_all(t):
28
+ data = d
29
+
30
+ n_data = {}
31
+ for k in data.keys():
32
+ c = extract(data, k)
33
+ n_data[k] = c
34
+ if n_data["globals"].get("memory", {}).get("path", None) is None:
35
+ if "memory" not in n_data["globals"]:
36
+ n_data["globals"]["memory"] = {}
37
+ n_data["globals"]["memory"]["path"] = path.parent / path.stem
38
+ CONFIG = BasicEducationRuntimeConfig(**n_data)
39
+ if CONFIG.evaluation is not None:
40
+ if CONFIG.evaluation.name is None:
41
+ CONFIG.evaluation.name = path.stem
42
+
src/elbench/basic_education_runtime/directions.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ from typing import Dict, Any, Tuple, Optional
3
+ from uuid import uuid4
4
+ from langgraph.graph.state import CompiledStateGraph
5
+ from langgraph.graph import StateGraph, START, END
6
+ from langgraph.prebuilt.chat_agent_executor import AgentState
7
+ from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
8
+
9
+ import aiosqlite
10
+
11
+ from elbench.basic_education_runtime.entity import AgentConfig
12
+ from elbench.basic_education_runtime.router import * # noqa: F403
13
+ from elbench.basic_education_runtime.config import CONFIG
14
+
15
+
16
+ def add_node_to_graph(graph: StateGraph, node_id: str, node_instance: Any) -> None:
17
+ if node_id not in graph.nodes:
18
+ graph.add_node(node_id, node_instance)
19
+ else:
20
+ pass
21
+
22
+
23
+ async def apply_agent_direction_from_dict(
24
+ agent_map: Dict[str, Tuple[CompiledStateGraph, AgentConfig]],
25
+ memory_id: Optional[str] = None,
26
+ task: Optional[Dict[str, str]] = None,
27
+ ) -> Tuple[CompiledStateGraph, str]:
28
+ if memory_id is None:
29
+ memory_id = str(uuid4())
30
+ directions = CONFIG.directions
31
+ graph = StateGraph(AgentState)
32
+ for node_id, node_instance in agent_map.items():
33
+ graph.add_node(node_id, node_instance[0])
34
+
35
+ for direction in directions:
36
+ start_node, end_node = (i.strip() for i in direction.split("->"))
37
+ if start_node == "START":
38
+ start_node = START
39
+ if end_node.startswith("router:"):
40
+ function_call = end_node[len("router:") :]
41
+ route, path_map = eval(function_call)
42
+ end_node = end_node.replace(":", "_")
43
+ graph.add_conditional_edges(start_node, route, path_map)
44
+ continue
45
+ # add_node_to_graph(graph, end_node, route)
46
+ elif end_node == "END":
47
+ end_node = END
48
+ else:
49
+ agent_config = agent_map[end_node][1]
50
+ pregel_instance = agent_map[end_node][0]
51
+ if not pregel_instance or not agent_config:
52
+ raise ValueError(f"Invalid configuration for {end_node}.")
53
+ graph.add_edge(start_node, end_node)
54
+ CONFIG.globals.memory.path.mkdir(parents=True, exist_ok=True)
55
+ path = CONFIG.globals.memory.path / f"{memory_id}.db"
56
+ # conn = sqlite3.connect(f"{memory_id}.db", check_same_thread=False)
57
+ if task is not None:
58
+ async with aiosqlite.connect(path) as conn:
59
+ sql = "create table task (key TEXT, value TEXT)"
60
+ await conn.execute(sql)
61
+ sql = "insert into task (key, value) values (?, ?)"
62
+ for key, value in task.items():
63
+ await conn.execute(sql, (key, value))
64
+ await conn.commit()
65
+ # await conn.close()
66
+ conn = aiosqlite.connect(path, check_same_thread=False)
67
+ if not hasattr(conn, "is_alive"):
68
+ conn.is_alive = lambda: bool( # type: ignore[attr-defined]
69
+ getattr(conn, "_running", False) and getattr(conn, "_connection", None) is not None
70
+ )
71
+ CONFIG.context.conns.append(conn)
72
+
73
+ memory = AsyncSqliteSaver(conn)
74
+ return graph.compile(checkpointer=memory), memory_id
75
+
76
+
77
+ apply_agent_direction = apply_agent_direction_from_dict
78
+
src/elbench/basic_education_runtime/entity.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import re
3
+ from typing import Dict, Any, Literal, Optional, List, Annotated, Tuple, Final
4
+ from pydantic import BaseModel, ConfigDict, Field, create_model
5
+ from typing_extensions import TypedDict
6
+ from langgraph.graph.message import add_messages
7
+ from pathlib import Path
8
+ from aiosqlite import Connection
9
+ from polyfactory.factories.pydantic_factory import ModelFactory
10
+ import math
11
+
12
+
13
+ # Common
14
+ class State(TypedDict):
15
+ messages: Annotated[list, add_messages]
16
+
17
+
18
+ # Memory
19
+ class Memory(BaseModel):
20
+ path: Path = Path(".")
21
+
22
+
23
+ # RetryConfig
24
+ class RetryConfig(BaseModel):
25
+ attempt: int = 3
26
+ interval: int = 3
27
+
28
+
29
+ # Global
30
+ class GlobalConfig(BaseModel):
31
+ concurrency: int = 8
32
+ recursion_limit: int = 25
33
+ memory: Memory = Memory()
34
+ retry: RetryConfig = RetryConfig()
35
+ model_call_timeout_seconds: int = 600
36
+
37
+
38
+ # Model
39
+ class ModelConfig(BaseModel):
40
+ api_base: Optional[str]
41
+ api_key: Optional[str]
42
+ kargs: Optional[Dict[str, Any]] = None
43
+ model: Optional[str]
44
+ type: str = "openai"
45
+ # Which part of the run this model plays (target, student, judge); used only for spending limits.
46
+ role: Optional[str] = None
47
+
48
+
49
+ # Agent
50
+ class Prompt(BaseModel):
51
+ role: Optional[str]
52
+ content: str
53
+
54
+
55
+ class SwitchConfig(BaseModel):
56
+ swap_user_assistant: bool = True
57
+
58
+
59
+ class AgentMemoryConfig(BaseModel):
60
+ enable: bool = True
61
+ id: Optional[str] = None
62
+ keep_turns: int = 3
63
+ # when_switch: SwitchConfig = SwitchConfig()
64
+
65
+
66
+ class AgentConfig(BaseModel):
67
+ model: str
68
+ prompt: Final[List[Prompt]]
69
+ memory: AgentMemoryConfig = AgentMemoryConfig(enable=True)
70
+
71
+ checkpointer: Optional[Any] = None
72
+
73
+
74
+ # Task
75
+ class TaskConfig(BaseModel):
76
+ start_prompt: Optional[Prompt] = None
77
+ variables: List[Dict[str, str]] = []
78
+
79
+ def model_post_init(self, __context):
80
+ object.__setattr__(self, "_frozen_start_prompt", True)
81
+
82
+ def __setattr__(self, name, value):
83
+ if getattr(self, f"_frozen_{name}", False) and name == "start_prompt":
84
+ raise AttributeError(f"{name} is const and cannot be modified")
85
+ super().__setattr__(name, value)
86
+
87
+
88
+ # Runtime Context
89
+ class BasicEducationRuntimeContext(BaseModel):
90
+ model_config = ConfigDict(arbitrary_types_allowed=True)
91
+ conns: List[Connection] = []
92
+
93
+
94
+ # ExportFormat
95
+ class ExportFormat(BaseModel):
96
+ task: Dict[str, str]
97
+ messages: List[Prompt] = []
98
+
99
+ @staticmethod
100
+ def from_json_file(file_path: Path | str) -> "ExportFormat":
101
+ with open(file_path, "r", encoding="utf-8") as f:
102
+ data = json.load(f)
103
+ return ExportFormat(**data)
104
+
105
+ def message_function(self, function_call: str) -> str:
106
+ if function_call == "as_dialog()":
107
+ strings = []
108
+ for message in self.messages:
109
+ if message.role is None:
110
+ continue
111
+ strings.append(f"{message.role}: {message.content}")
112
+ return "\n".join(strings)
113
+ else:
114
+ raise ValueError(f"Unsupported function call: {function_call}")
115
+
116
+ def replace_template(self, template: str) -> str:
117
+ """
118
+ 将输入的模板字符串中的占位符替换为实际值
119
+ 目前支持的占位符有:
120
+ task.xxxx 代表task的某个字段
121
+ messages.as_dialog() 代表对话形式的messages
122
+
123
+ messages.as_dialog() 形式如下:
124
+ teacher: xxxxxxx
125
+ student: xxxxxxx
126
+ """
127
+ # Only runtime placeholders should be substituted. Model outputs may
128
+ # legitimately contain braces such as "{1}", which must remain literal.
129
+ placeholders = re.findall(r"\{(?:task\.[^{}]+|messages\.[^{}]+)\}", template)
130
+ for placeholder in placeholders:
131
+ # 获取占位符的名称,即{}中的内容
132
+ placeholder_name = placeholder.strip("{}")
133
+ # 获取占位符对应的值,根据占位符的名称从task或messages中获取
134
+ if placeholder_name.startswith("task."):
135
+ field_name = placeholder_name.split(".")[1]
136
+ value = self.task[field_name]
137
+ elif placeholder_name.startswith("messages."):
138
+ message_name = placeholder_name.split(".")[1]
139
+ # 如果是函数调用形式
140
+ if "(" in message_name:
141
+ value = self.message_function(message_name)
142
+ else:
143
+ raise Exception(f"Invalid message name: {message_name}")
144
+ else:
145
+ raise Exception(f"Invalid placeholder name: {placeholder_name}")
146
+ # 将占位符替换为对应的值
147
+ template = template.replace(placeholder, str(value))
148
+ return template
149
+
150
+
151
+ # FormatField
152
+ class FormatField(BaseModel):
153
+ field: str
154
+ type: Literal["str", "int", "float", "bool", "dict"]
155
+ description: str
156
+ items: List["FormatField"] = []
157
+ # min: float | int | None = None
158
+ max: float | int | None = None
159
+
160
+
161
+ # Evaluation
162
+ class EvalConfig(BaseModel):
163
+ model: str
164
+ name: Optional[str]
165
+ prompt: List[Prompt]
166
+ format: List[FormatField]
167
+ format_mode: Literal["tool", "prompt"] = "tool"
168
+
169
+ def format_to_json_schema(self) -> str:
170
+ model = self.format_to_pydantic()
171
+ json_schema = model.model_json_schema()
172
+ return json.dumps(json_schema, ensure_ascii=False)
173
+
174
+ def format_to_json_example(self) -> str:
175
+ mmodel: type[BaseModel] = self.format_to_pydantic()
176
+
177
+ class MMF(ModelFactory):
178
+ __model__ = mmodel
179
+
180
+ return MMF().build().json()
181
+
182
+ def get_prompts(self) -> Tuple[str, List[Prompt]]:
183
+ """获取系统提示和其他提示词"""
184
+ system_prompt = ""
185
+ other_prompt: List[Prompt] = []
186
+ for p in self.prompt:
187
+ if p.role == "system":
188
+ if system_prompt != "":
189
+ raise ValueError("存在多个系统提示")
190
+ system_prompt = p.content
191
+ else:
192
+ other_prompt.append(p)
193
+ return system_prompt, other_prompt
194
+
195
+ def format_to_pydantic(self) -> type[BaseModel]:
196
+ def field_type_from_format(f: FormatField) -> tuple:
197
+ """将FormatField转成pydantic字段元组(类型,Field信息)"""
198
+ python_type_map = {
199
+ "str": str,
200
+ "int": int,
201
+ "float": float,
202
+ "bool": bool,
203
+ }
204
+
205
+ if f.type == "dict":
206
+ nested_model = build_model_from_format(f.items, f.field)
207
+ return nested_model, Field(..., description=f.description)
208
+ if f.type == "int" or f.type == "float":
209
+ py_type = python_type_map[f.type]
210
+ # if f.min is None:
211
+ # if f.max is None:
212
+ # return py_type, Field(..., description=f.description)
213
+ # else:
214
+ # return py_type, Field(..., le=f.max, description=f.description)
215
+ # else:
216
+ # if f.max is None:
217
+ # return py_type, Field(..., ge=f.min, description=f.description)
218
+ # else:
219
+ # return py_type, Field(..., ge=f.min, le=f.max, description=f.description)
220
+
221
+ if f.max is None:
222
+ return py_type, Field(..., description=f.description)
223
+ else:
224
+ return py_type, Field(..., le=f.max, description=f.description)
225
+ else:
226
+ py_type = python_type_map[f.type]
227
+ return py_type, Field(..., description=f.description)
228
+
229
+ def build_model_from_format(
230
+ fields: List[FormatField], model_name: str = "DynamicModel"
231
+ ) -> type[BaseModel]:
232
+ annotations = {}
233
+ for f in fields:
234
+ annotations[f.field] = field_type_from_format(f)
235
+ return create_model(model_name, **annotations)
236
+
237
+ return build_model_from_format(self.format, "GeneratedModel")
238
+
239
+
240
+ # Runtime
241
+ class BasicEducationRuntimeConfig(BaseModel):
242
+ globals: GlobalConfig
243
+ models: Dict[str, ModelConfig]
244
+ agents: Dict[str, AgentConfig]
245
+ directions: List[str]
246
+ tasks: TaskConfig
247
+ evaluation: Optional[EvalConfig] = None
248
+
249
+ context: BasicEducationRuntimeContext = BasicEducationRuntimeContext(conns=[])
src/elbench/basic_education_runtime/evaluation.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import asyncio
3
+ from typing import Any, Dict
4
+ from langchain_core.tools import tool, BaseTool
5
+ from elbench.basic_education_runtime.config import CONFIG
6
+ from langgraph.prebuilt import create_react_agent
7
+ from langchain.chat_models.base import BaseChatModel
8
+ from elbench.basic_education_runtime.entity import ExportFormat
9
+ from pydantic import BaseModel
10
+ from tenacity import retry, stop_after_attempt, wait_fixed
11
+ import json
12
+
13
+
14
+ def generate_evaluation_tool() -> BaseTool:
15
+ """
16
+ Generates a tool for evaluating the basic education runtime outputs.
17
+ This function is a placeholder and should be implemented with actual logic.
18
+ """
19
+
20
+ if CONFIG.evaluation is None:
21
+ raise ValueError("Evaluation configuration not found.")
22
+
23
+ @tool(
24
+ name_or_callable="save_result_to_database",
25
+ description="Save the evaluation results to a database.",
26
+ return_direct=True,
27
+ args_schema=CONFIG.evaluation.format_to_pydantic(),
28
+ )
29
+ def save_to_db(**kwargs):
30
+ """
31
+ Save the evaluation results to a database.
32
+ """
33
+ for k, v in kwargs.items():
34
+ if issubclass(v.__class__, BaseModel):
35
+ v = v.model_dump()
36
+ kwargs[k] = v
37
+ return kwargs
38
+
39
+ return save_to_db
40
+
41
+
42
+ def _build_evaluation_messages(
43
+ *,
44
+ system_prompt: str,
45
+ other_prompts: list[Any],
46
+ exported_result: ExportFormat,
47
+ ) -> list[dict[str, str]]:
48
+ messages: list[dict[str, str]] = []
49
+ for op in other_prompts:
50
+ content = exported_result.replace_template(op.content)
51
+ role = str(op.role or "user").strip() or "user"
52
+ messages.append(
53
+ {
54
+ "role": role,
55
+ "content": content,
56
+ }
57
+ )
58
+
59
+ has_user_query = any(
60
+ item.get("role") == "user" and str(item.get("content", "")).strip()
61
+ for item in messages
62
+ )
63
+ if not has_user_query:
64
+ fallback = exported_result.replace_template(system_prompt).strip()
65
+ if not fallback:
66
+ fallback = "Please complete the evaluation."
67
+ messages.append(
68
+ {
69
+ "role": "user",
70
+ "content": fallback,
71
+ }
72
+ )
73
+ return messages
74
+
75
+
76
+ @retry(
77
+ stop=stop_after_attempt(CONFIG.globals.retry.attempt),
78
+ wait=wait_fixed(CONFIG.globals.retry.interval),
79
+ )
80
+ async def evaluate(
81
+ model: BaseChatModel, exported_result: ExportFormat
82
+ ) -> Dict[str, Any]:
83
+ assert CONFIG.evaluation
84
+ system_prompt, other_prompt = CONFIG.evaluation.get_prompts()
85
+ system_prompt = exported_result.replace_template(system_prompt)
86
+ ops = _build_evaluation_messages(
87
+ system_prompt=system_prompt,
88
+ other_prompts=other_prompt,
89
+ exported_result=exported_result,
90
+ )
91
+ mc = getattr(model, "model_config", None)
92
+ if getattr(mc, "type", None) == "system_one":
93
+ from elbench.judges.system_one_judge import evaluate_basic_education_system_one
94
+
95
+ return await evaluate_basic_education_system_one(
96
+ api_base=mc.api_base,
97
+ api_key=mc.api_key,
98
+ model_name=str(mc.model or "liquid/d1-ultra"),
99
+ timeout_seconds=float(CONFIG.globals.model_call_timeout_seconds or 60.0),
100
+ retry_attempts=int(CONFIG.globals.retry.attempt or 1),
101
+ system_prompt=system_prompt,
102
+ messages=ops,
103
+ format_fields=list(CONFIG.evaluation.format),
104
+ )
105
+ if CONFIG.evaluation.format_mode == "tool":
106
+ tools = [generate_evaluation_tool()]
107
+
108
+ # other_prompt = exported_result.replace_template(other_prompt)
109
+ agent = create_react_agent(
110
+ model=model.bind_tools(
111
+ tools,
112
+ tool_choice={
113
+ "type": "function",
114
+ "function": {"name": "save_result_to_database"},
115
+ }, # type: ignore
116
+ # tool_choice="required",
117
+ ),
118
+ tools=tools,
119
+ # model=model,
120
+ prompt=system_prompt
121
+ + "\n\nPlease call the save_result_to_database tool to store the evaluation result.",
122
+ )
123
+
124
+ a = await asyncio.wait_for(
125
+ agent.ainvoke({"messages": ops}),
126
+ timeout=CONFIG.globals.model_call_timeout_seconds,
127
+ )
128
+ data = json.loads(a["messages"][-1].content)
129
+ return data
130
+ elif CONFIG.evaluation.format_mode == "prompt":
131
+ format_example = CONFIG.evaluation.format_to_json_example()
132
+ agent = create_react_agent(
133
+ model=model,
134
+ tools=[],
135
+ prompt=system_prompt
136
+ + "\n\nReturn exactly one JSON object wrapped with <START OF EVAL OUTPUT> and <END OF EVAL OUTPUT>."
137
+ + "\nDo not add markdown fences, explanations, or extra text."
138
+ + "\nUse the same keys and value types as this example:\n"
139
+ + "\n<START OF EVAL OUTPUT>\n"
140
+ + f"{format_example}\n"
141
+ + "<END OF EVAL OUTPUT>",
142
+ )
143
+
144
+ a = await asyncio.wait_for(
145
+ agent.ainvoke({"messages": ops}),
146
+ timeout=CONFIG.globals.model_call_timeout_seconds,
147
+ )
148
+ response: str = a["messages"][-1].content
149
+
150
+ # Fuck Gemini
151
+ response = response.replace("\\<", "<")
152
+ response = response.replace("\\>", ">")
153
+
154
+ # print(response)
155
+
156
+ # 姝e垯琛ㄨ揪寮忓尮閰?START OUTPUT>鍜?END OUTPUT>
157
+ match = re.findall(
158
+ r"<START OF EVAL OUTPUT>(.*)<END OF EVAL OUTPUT>", response, re.DOTALL
159
+ )
160
+ if len(match) > 0:
161
+ text = match[-1]
162
+ # 濡傛灉text琚玚``鍖呭洿锛屽幓鎺?
163
+ text = text.strip().strip("```").strip("json")
164
+ # print(text, "#############")
165
+ return json.loads(text)
166
+ else:
167
+ raise ValueError("Output does not contain <START OUTPUT> and <END OUTPUT>")
168
+ else:
169
+ raise ValueError(f"Invalid format mode: {CONFIG.evaluation.format_mode}")
170
+
171
+
172
+ if __name__ == "__main__":
173
+ import asyncio
174
+
175
+ async def main():
176
+ from elbench.basic_education_runtime.model import init_chat_model_from_dict
177
+ # from langchain.globals import set_debug
178
+
179
+ # set_debug(True)
180
+
181
+ ef = ExportFormat.from_json_file(
182
+ "/tmp/basic_education_runtime/sample.json"
183
+ )
184
+
185
+ model = init_chat_model_from_dict(CONFIG.models["4o_teacher"])
186
+ a = await evaluate(model, ef)
187
+ print(a)
188
+
189
+ asyncio.run(main())
190
+
src/elbench/basic_education_runtime/langchain_compat.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Compatibility helpers for LangChain version differences."""
2
+
3
+ try:
4
+ from langchain.globals import set_debug
5
+ except ModuleNotFoundError:
6
+ try:
7
+ from langchain_core.globals import set_debug
8
+ except ModuleNotFoundError:
9
+ def set_debug(_: bool) -> None:
10
+ """Fallback when LangChain debug globals are unavailable."""
11
+
12
+ return None
src/elbench/basic_education_runtime/model.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any, Dict
4
+
5
+ from langchain.chat_models import init_chat_model
6
+ from langchain.chat_models.base import BaseChatModel
7
+ from langchain_core.messages import AIMessage
8
+
9
+ from elbench.basic_education_runtime.config import CONFIG
10
+ from elbench.basic_education_runtime.entity import ModelConfig
11
+ from elbench.basic_education_runtime.spend_callback import SpendCallback
12
+ from elbench.utils import spend
13
+
14
+
15
+ class MockRuntimeChatModel:
16
+ def __init__(self, model_config: ModelConfig) -> None:
17
+ self.model_config = model_config
18
+
19
+ async def ainvoke(self, messages: list[Any]) -> AIMessage:
20
+ prefix = ""
21
+ if self.model_config.kargs:
22
+ prefix = str(self.model_config.kargs.get("prefix", "")).strip()
23
+
24
+ last_content = ""
25
+ for item in reversed(messages):
26
+ content = getattr(item, "content", None)
27
+ if isinstance(content, str) and content.strip():
28
+ last_content = content.strip()
29
+ break
30
+ if isinstance(item, dict):
31
+ dict_content = item.get("content")
32
+ if isinstance(dict_content, str) and dict_content.strip():
33
+ last_content = dict_content.strip()
34
+ break
35
+
36
+ summary = last_content[:120]
37
+ content = "占位回答:我们一步一步来。<end>"
38
+ if summary:
39
+ content = f"{prefix} 占位回答:已收到问题。{summary}\n我们一步一步来。<end>".strip()
40
+ elif prefix:
41
+ content = f"{prefix} {content}".strip()
42
+ return AIMessage(content=content)
43
+
44
+ def bind_tools(self, tools: list[Any], tool_choice: Any = None) -> "MockRuntimeChatModel":
45
+ return self
46
+
47
+
48
+ class SystemOneRuntimeModel:
49
+ def __init__(self, model_config: ModelConfig) -> None:
50
+ self.model_config = model_config
51
+
52
+
53
+ def init_chat_model_from_dict(mc: ModelConfig) -> BaseChatModel:
54
+ if mc.type == "mock":
55
+ return MockRuntimeChatModel(mc) # type: ignore[return-value]
56
+ if mc.type == "system_one":
57
+ return SystemOneRuntimeModel(mc) # type: ignore[return-value]
58
+ if mc.kargs is not None:
59
+ llm = init_chat_model(
60
+ model=f"{mc.type}:{mc.model}",
61
+ api_key=mc.api_key,
62
+ base_url=mc.api_base,
63
+ **mc.kargs,
64
+ )
65
+ else:
66
+ llm = init_chat_model(
67
+ model=f"{mc.type}:{mc.model}",
68
+ api_key=mc.api_key,
69
+ base_url=mc.api_base,
70
+ )
71
+ if mc.role and spend.active_ledger() is not None:
72
+ llm.callbacks = [*(llm.callbacks or []), SpendCallback(mc.role)]
73
+ return llm
74
+
75
+
76
+ def init_model_map_from_dict() -> Dict[str, BaseChatModel]:
77
+ cfg = CONFIG.models
78
+ result = {}
79
+ for k, v in cfg.items():
80
+ result[k] = init_chat_model_from_dict(v)
81
+ return result
82
+
83
+
84
+ init_model_map = init_model_map_from_dict
85
+
86
+ if __name__ == "__main__":
87
+ a = init_model_map_from_dict()
88
+ print(a)