Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +6 -0
- .gitignore +40 -0
- .python-version +1 -0
- AGENTS.md +71 -0
- LICENSE +21 -0
- Makefile +32 -0
- README.md +71 -0
- experiments/analysis/README.md +66 -0
- experiments/analysis/checks.md +33 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q4-mtp.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q5-mtp.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-gemma-q8-mtp.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dflash.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-eagle3.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dflash.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q5-eagle3.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv +10 -0
- experiments/analysis/curves/acc_by_pos_curves-qwen-q8-eagle3.csv +10 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv +22 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv +31 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv +16 -0
- experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv +22 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
paper/figures/F1a_acceptance_qwen.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
paper/figures/F1b_acceptance_gemma.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
paper/figures/F2_speedup.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
paper/figures/F3_breakeven.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
paper/figures/F4_alpha_position_ksweep.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
paper/paper.pdf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# modelos grandes y artefactos pesados (se bajan por script, no se commitean)
|
| 2 |
+
models/
|
| 3 |
+
*.gguf
|
| 4 |
+
*.safetensors
|
| 5 |
+
*.bin
|
| 6 |
+
|
| 7 |
+
# papers y notas del corpus: se regeneran con scripts/download_papers.py, no se versionan
|
| 8 |
+
papers/
|
| 9 |
+
notes/
|
| 10 |
+
scripts/download_papers.py
|
| 11 |
+
|
| 12 |
+
# datos de experimentos crudos (se regeneran; los resultados limpios sí se commitean)
|
| 13 |
+
experiments/runs/
|
| 14 |
+
experiments/prompts/raw/
|
| 15 |
+
|
| 16 |
+
# python (uv)
|
| 17 |
+
.venv/
|
| 18 |
+
__pycache__/
|
| 19 |
+
*.pyc
|
| 20 |
+
|
| 21 |
+
# entorno local (no parte del proyecto)
|
| 22 |
+
.atl/
|
| 23 |
+
|
| 24 |
+
# el paper (manuscript/) y la documentación interna de investigación (docs/) no se versionan:
|
| 25 |
+
# el paper se publica por separado (arXiv/HF) y el repo público es su companion
|
| 26 |
+
manuscript/
|
| 27 |
+
docs/
|
| 28 |
+
|
| 29 |
+
# estado vivo del proyecto (bitácora local, no se versiona)
|
| 30 |
+
STATUS.md
|
| 31 |
+
|
| 32 |
+
# tests locales (no se versionan)
|
| 33 |
+
tests/
|
| 34 |
+
|
| 35 |
+
# editores/OS
|
| 36 |
+
.DS_Store
|
| 37 |
+
*.swp
|
| 38 |
+
tests/test_smoke.py
|
| 39 |
+
tests/conftest.py
|
| 40 |
+
tests/
|
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.12
|
AGENTS.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AGENTS.md — Rules for agents (and humans) working in this repo
|
| 2 |
+
|
| 3 |
+
This repository is a **research project** with a master plan and reproducible
|
| 4 |
+
protocol. Before touching code or experiments, read this completely.
|
| 5 |
+
|
| 6 |
+
## 1. Context in one sentence
|
| 7 |
+
|
| 8 |
+
Empirical study of modern speculative decoding (EAGLE-3/DFlash/DSpark) on an
|
| 9 |
+
RTX 4060 Ti 16 GB with Gemma 4 12B and Qwen3-8B GGUF: acceptance by domain,
|
| 10 |
+
quantization and hardware break-even. The research documentation (thesis, gaps,
|
| 11 |
+
plan) lives locally in `docs/` (not versioned); live status is ALWAYS in
|
| 12 |
+
`STATUS.md` (local, not versioned).
|
| 13 |
+
|
| 14 |
+
## 2. Conventions
|
| 15 |
+
|
| 16 |
+
- **Language**: docs, logs and commit messages in **English**; code and
|
| 17 |
+
technical metadata in **English**. Do not mix languages within one file.
|
| 18 |
+
- **Documentation**: markdown and session logs in `docs/`, live status in
|
| 19 |
+
`STATUS.md` — all local, **not versioned**. The corpus `papers/` and its
|
| 20 |
+
notes `notes/` are not versioned (regenerated locally with a download script,
|
| 21 |
+
outside the repo).
|
| 22 |
+
- **Environment**: activate with `source scripts/spec-env.sh` (uv + CUDA in
|
| 23 |
+
`~/cuda`). Never install anything with sudo; if privileges are needed, ask
|
| 24 |
+
uli and wait.
|
| 25 |
+
- **Minimum quality before commit**: `ruff check`, `ruff format --check`,
|
| 26 |
+
`pyright` with no errors on what you touched.
|
| 27 |
+
|
| 28 |
+
## 3. Git rules
|
| 29 |
+
|
| 30 |
+
- **Logical commits** (never a mega-commit); concise messages in English with a
|
| 31 |
+
type prefix (`docs:`, `chore:`, `feat:`, `fix:`).
|
| 32 |
+
- **Do not version**: `models/`, `*.gguf`, `papers/`, `notes/`,
|
| 33 |
+
`experiments/runs/`, `.venv/`, `.atl/` (see `.gitignore`).
|
| 34 |
+
- Do not commit regenerable artifacts (PDFs, raw runs, build logs).
|
| 35 |
+
- Before committing: `git status` + `git diff` to review what goes in.
|
| 36 |
+
|
| 37 |
+
## 4. Experiments (Phase 1+)
|
| 38 |
+
|
| 39 |
+
- **Never** launch a run without a viable `--resume` (incremental JSONL) unless
|
| 40 |
+
explicitly decided.
|
| 41 |
+
- Follow the master plan protocol (§4): fixed seeds (42), documented temperature,
|
| 42 |
+
target-only baseline with the SAME quantization, config.json + metrics.json per
|
| 43 |
+
run in `experiments/runs/<id>/`.
|
| 44 |
+
- **Do not** use the full DeepSpec data cache (~38 TB); only subsamples.
|
| 45 |
+
- Do not change the master plan targets/decisions (§10) without uli's approval.
|
| 46 |
+
|
| 47 |
+
## 5. Sessions and handoff (mandatory)
|
| 48 |
+
|
| 49 |
+
At the end of any work session:
|
| 50 |
+
|
| 51 |
+
1. Write the session log at `docs/logs/YYYY-MM-DD-sesion-NN-asunto.md` (local
|
| 52 |
+
logbook, not versioned: what was done, decisions, pending items, what is
|
| 53 |
+
still running).
|
| 54 |
+
2. Update `STATUS.md` (local, not versioned: current phase, what is running,
|
| 55 |
+
pending, next steps).
|
| 56 |
+
3. Save in engram (`mem_save`/`mem_session_summary`) non-obvious decisions and
|
| 57 |
+
discoveries (project: `spectulative_decoding`).
|
| 58 |
+
4. If background processes remain (downloads, builds, runs): note them in
|
| 59 |
+
STATUS.md (local) with PID and log.
|
| 60 |
+
|
| 61 |
+
A new agent opening the repo should read: `AGENTS.md` → (local `STATUS.md`,
|
| 62 |
+
docs and logs) before acting.
|
| 63 |
+
|
| 64 |
+
## 6. Forbidden / careful
|
| 65 |
+
|
| 66 |
+
- **Forbidden**: sudo without notice; versioning models/PDFs/runs; using the
|
| 67 |
+
38 TB cache; modifying raw results; deleting another session's
|
| 68 |
+
`experiments/runs/`.
|
| 69 |
+
- **Careful**: recompiling llama.cpp without noting the exact commit; changing
|
| 70 |
+
the overnight/resume regime; discarding speedup < 1 results (they are valid
|
| 71 |
+
data, not errors).
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Speculative Decoding Lab contributors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
Makefile
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Makefile — common lab tasks
|
| 2 |
+
# Requires uv in PATH (~/.local/bin/uv).
|
| 3 |
+
|
| 4 |
+
.PHONY: setup lint fmt typecheck models status clean
|
| 5 |
+
|
| 6 |
+
setup: ## Full environment (venv + CUDA toolkit)
|
| 7 |
+
uv sync
|
| 8 |
+
bash scripts/prepare-cuda.sh || true
|
| 9 |
+
|
| 10 |
+
lint: ## ruff check
|
| 11 |
+
uv run ruff check scripts/
|
| 12 |
+
|
| 13 |
+
fmt: ## ruff format (applies)
|
| 14 |
+
uv run ruff format scripts/
|
| 15 |
+
|
| 16 |
+
fmt-check: ## ruff format --check
|
| 17 |
+
uv run ruff format --check scripts/
|
| 18 |
+
|
| 19 |
+
typecheck: ## pyright
|
| 20 |
+
uv run pyright scripts/
|
| 21 |
+
|
| 22 |
+
models: ## List available GGUFs in models/
|
| 23 |
+
ls -lh models/
|
| 24 |
+
|
| 25 |
+
status: ## Live project status
|
| 26 |
+
@cat STATUS.md
|
| 27 |
+
|
| 28 |
+
clean: ## Remove regenerable artifacts (does NOT delete models/)
|
| 29 |
+
rm -rf experiments/runs/*
|
| 30 |
+
|
| 31 |
+
help:
|
| 32 |
+
@grep -E '^[a-zA-Z-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}'
|
README.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Speculative Decoding Lab
|
| 2 |
+
|
| 3 |
+
Systematic, reproducible empirical evaluation of **modern speculative decoding**
|
| 4 |
+
(EAGLE-3, DFlash, DSpark vs. vanilla and model-free) on an **RTX 4060 Ti 16 GB**
|
| 5 |
+
with **Gemma 4 12B** and **Qwen3-8B** GGUF-quantized models: acceptance by
|
| 6 |
+
cognitive domain, quantization effects, and hardware break-even maps.
|
| 7 |
+
|
| 8 |
+
> Paper: [PDF](paper/paper.pdf) · [arXiv-ready source bundle](paper/arxiv-bundle.tar.gz) · [source (markdown)](paper/paper.md)
|
| 9 |
+
|
| 10 |
+
## Hardware used
|
| 11 |
+
|
| 12 |
+
The study was developed and benchmarked on this machine, so all reported
|
| 13 |
+
numbers (speedups, acceptance, break-even) are tied to these specs. The code
|
| 14 |
+
itself is portable: it only needs llama.cpp (CUDA build) and enough VRAM for
|
| 15 |
+
the target model, and can be re-run on any other hardware.
|
| 16 |
+
|
| 17 |
+
| Component | Detail |
|
| 18 |
+
| --- | --- |
|
| 19 |
+
| GPU | NVIDIA RTX 4060 Ti **16 GB** (Ada Lovelace, compute 8.9) |
|
| 20 |
+
| CPU | Intel i5-12400 (6C/12T, 4.4 GHz) |
|
| 21 |
+
| RAM / Disk | 31 GB / ~185 GB free |
|
| 22 |
+
| OS | Ubuntu 24.04 · driver 595.84 |
|
| 23 |
+
|
| 24 |
+
Environment principle: **everything user-space, no sudo** (uv + CUDA runfile in `$HOME`).
|
| 25 |
+
|
| 26 |
+
## Stack
|
| 27 |
+
|
| 28 |
+
- **Python 3.12 + uv** — `pyproject.toml` + `uv.lock` (numpy, ruff, pyright, cmake, ninja).
|
| 29 |
+
- **CUDA toolkit 12.8.1** in `~/cuda` (runfile `--toolkitpath`). *Note: the `nvcc` binary is not on PyPI (the wheels only ship ptxas).*
|
| 30 |
+
- **llama.cpp master 0713275 (v22)** (CUDA build) in `~/llama.cpp` (built from source; no CUDA binaries are published for Linux).
|
| 31 |
+
- **GGUF models** in `models/` (not versioned; downloaded with `curl -C -`).
|
| 32 |
+
|
| 33 |
+
## Setup (once)
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
uv sync # create .venv with all dependencies
|
| 37 |
+
source scripts/spec-env.sh # PATH + CUDA_HOME + env vars (idempotent)
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
## Quality (required before committing)
|
| 41 |
+
|
| 42 |
+
```bash
|
| 43 |
+
uv run ruff check . # lint
|
| 44 |
+
uv run ruff format --check . # formatting
|
| 45 |
+
uv run pyright # types
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
## Execution regime
|
| 49 |
+
|
| 50 |
+
Overnight sessions with **pause/resume**: `scripts/bench_accept.py --resume`
|
| 51 |
+
(incremental JSONL per prompt; server-based runner with α/τ), downloads with
|
| 52 |
+
`scripts/dl.py`, runs in tmux/nohup. Measurement campaigns are launched with
|
| 53 |
+
`scripts/run_final_chain.sh`, `scripts/run_curves_chain.sh` and
|
| 54 |
+
`scripts/run_ksweep_chain.sh`; final analysis with `scripts/analyze_final.py`.
|
| 55 |
+
|
| 56 |
+
## Structure
|
| 57 |
+
|
| 58 |
+
```
|
| 59 |
+
scripts/ environment, downloads, benchmark runner and analysis (bench_accept.py, analyze_final.py)
|
| 60 |
+
experiments/ runs/ (raw, not versioned) and prompts/ (JSONL subsamples)
|
| 61 |
+
models/ GGUFs (not versioned)
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
## For agents
|
| 65 |
+
|
| 66 |
+
Read [`AGENTS.md`](AGENTS.md) before working in this repo (conventions, git
|
| 67 |
+
rules, sessions and handoff).
|
| 68 |
+
|
| 69 |
+
## License
|
| 70 |
+
|
| 71 |
+
MIT — see [`LICENSE`](LICENSE).
|
experiments/analysis/README.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Final analysis - `experiments/analysis/`
|
| 2 |
+
|
| 3 |
+
Generated by `scripts/analyze_final.py` (Python 3.12, stdlib only + numpy + matplotlib).
|
| 4 |
+
|
| 5 |
+
## Methodology
|
| 6 |
+
|
| 7 |
+
### Sources
|
| 8 |
+
|
| 9 |
+
- `experiments/runs/<run>/results.jsonl` - one OK record per line (schema in the
|
| 10 |
+
inventory). `config.json` normalizes the run: family (`final/curves/ksweep`),
|
| 11 |
+
target (`{qwen|gemma}-{q4|q5|q8}` from the model path), drafter from
|
| 12 |
+
`spec_type` + `p_min` (dspark) + draft quantization (dflash-f16/q4/q8).
|
| 13 |
+
- `server.log` from `curves-*`/`ksweep-*` runs - `draft acceptance` and
|
| 14 |
+
`acc per pos` lines (only place with per-position curves; the `final-*` runs
|
| 15 |
+
rotated logs and their original segment has no acceptance lines).
|
| 16 |
+
|
| 17 |
+
### Exclusions (timing sentinels)
|
| 18 |
+
|
| 19 |
+
llama-server reports `tok_per_s = 1,000,000` / `predicted_ms = 0` in 271 records
|
| 20 |
+
from Gemma runs (timing quirk, not real speedup). **Rule applied in every
|
| 21 |
+
calculation**: exclude any record with `tok_per_s >= 1e5` or `predicted_ms <= 0`
|
| 22 |
+
(together with `alpha/tau/draft_n = None`). Qwen has no sentinels. The Gemma
|
| 23 |
+
`metrics.json` files are contaminated by this (mean 12,291 tok/s) and are **not**
|
| 24 |
+
used for tok/s; only `duration_s` and `errors` are used. Counts per run in
|
| 25 |
+
`summary.json` -> `meta.exclusions_per_run` (q4: 18/run, q5: 25/run, q8: 20/run
|
| 26 |
+
in final; 3-4/run in gemma curves/ksweep).
|
| 27 |
+
|
| 28 |
+
### Log <-> record mapping (curves/ksweep)
|
| 29 |
+
|
| 30 |
+
The order of the `draft acceptance` lines in `server.log` == order of records
|
| 31 |
+
with non-None `alpha` in `results.jsonl` (verified in 41/41 runs, max |diff| =
|
| 32 |
+
5e-5, rounding only). Gemma sentinel records (alpha=None) have no line and are
|
| 33 |
+
skipped. This assigns the domain to each line. No mismatches in any run.
|
| 34 |
+
|
| 35 |
+
### Speedup vs solo
|
| 36 |
+
|
| 37 |
+
Per-prompt ratio `tps_draft / tps_solo` matched by `id` against the `solo` run
|
| 38 |
+
of the same family+quant; mean and median of the ratio are reported, plus the
|
| 39 |
+
ratio of means (`agg`). The 6 solo runs (qwen q4/q5/q8, gemma q4/q5/q8) exist
|
| 40 |
+
and are the baseline.
|
| 41 |
+
|
| 42 |
+
### Break-even alpha_be (paper #32, Bielik et al.)
|
| 43 |
+
|
| 44 |
+
Per `ksweep-*` run and domain (and aggregated): OLS of `TPS = a + beta*alpha`
|
| 45 |
+
over the run records (alpha and per-prompt tok/s, clean records with alpha).
|
| 46 |
+
`alpha_be = (TPS_base - a) / beta` where `TPS_base` = mean tok/s of the
|
| 47 |
+
corresponding `solo` run (same family+quant+domain, final run). CI95 by delta
|
| 48 |
+
method over the OLS covariance of (a, beta). alpha_be > 1 = no reachable
|
| 49 |
+
break-even; < 0 = always above baseline. Paper #32 reports k=2: 38.0-52.8% and
|
| 50 |
+
k=4: 77.7-90.1% (Table 5); the compact 40-77% range covers k=2..4.
|
| 51 |
+
|
| 52 |
+
## Files
|
| 53 |
+
|
| 54 |
+
- `summary.json` - consolidated numbers (per config x domain and breakeven block).
|
| 55 |
+
- `tables/t2_speedup.md` ... `tables/t6_breakeven.md` - markdown tables for the paper.
|
| 56 |
+
- `curves/acc_by_pos_<run>.csv` - per-position acceptance per domain (41 runs).
|
| 57 |
+
- `checks.md` - result of the checks vs session 08.
|
| 58 |
+
- `manuscript/figures/F1..F4_*.png` - figures (300 dpi; local paper output, `manuscript/` is not versioned).
|
| 59 |
+
|
| 60 |
+
## Execution
|
| 61 |
+
|
| 62 |
+
```
|
| 63 |
+
.venv/bin/python scripts/analyze_final.py \
|
| 64 |
+
--runs-dir experiments/runs --out-dir experiments/analysis \
|
| 65 |
+
--figs-dir manuscript/figures
|
| 66 |
+
```
|
experiments/analysis/checks.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CHECKS - expected values (session 08) vs computed
|
| 2 |
+
|
| 3 |
+
Tolerance: >5% relative deviation -> warning. Speedups are compared against
|
| 4 |
+
the mean and median of the per-prompt ratio; either one passing is enough.
|
| 5 |
+
|
| 6 |
+
| Check | Expected | Computed | Pass? | Note |
|
| 7 |
+
| ------- | ---------- | ----------- | ------- | ------ |
|
| 8 |
+
| qwen-q4 solo | 53.400 | 53.368 | OK | |
|
| 9 |
+
| vanilla17b | 75.500 | 75.453 | OK | |
|
| 10 |
+
| vanilla17b x | 1.410 | 1.41 (med 1.41) | OK | mean 0.3% off, median 0.3% off |
|
| 11 |
+
| vanilla17b alpha | 0.725 | 0.725 | OK | |
|
| 12 |
+
| eagle3 | 74.400 | 74.371 | OK | |
|
| 13 |
+
| eagle3 x | 1.390 | 1.39 (med 1.37) | OK | mean 0.3% off, median 1.2% off |
|
| 14 |
+
| eagle3 alpha | 0.440 | 0.440 | OK | |
|
| 15 |
+
| dspark-p0 | 87.700 | 87.617 | OK | |
|
| 16 |
+
| dspark-p0 x | 1.640 | 1.64 (med 1.63) | OK | mean 0.1% off, median 0.8% off |
|
| 17 |
+
| dspark-p0 alpha | 0.616 | 0.616 | OK | |
|
| 18 |
+
| dspark-p6 | 80.800 | 80.731 | OK | |
|
| 19 |
+
| dspark-p6 x | 1.510 | 1.51 (med 1.50) | OK | mean 0.2% off, median 0.8% off |
|
| 20 |
+
| dspark-p6 alpha | 0.714 | 0.714 | OK | |
|
| 21 |
+
| qwen-q5 solo | 46.800 | 46.800 | OK | |
|
| 22 |
+
| q5 eagle3 | 68.300 | 68.331 | OK | |
|
| 23 |
+
| q5 eagle3 x | 1.460 | 1.46 (med 1.44) | OK | mean 0.0% off, median 1.4% off |
|
| 24 |
+
| q5 dspark-p0 | 80.700 | 80.641 | OK | |
|
| 25 |
+
| q5 dspark-p0 x | 1.720 | 1.72 (med 1.72) | OK | mean 0.2% off, median 0.3% off |
|
| 26 |
+
| qwen-q8 solo | 32.400 | 32.345 | OK | |
|
| 27 |
+
| q8 eagle3 | 52.800 | 52.812 | OK | |
|
| 28 |
+
| q8 eagle3 x | 1.630 | 1.63 (med 1.61) | OK | mean 0.2% off, median 1.0% off |
|
| 29 |
+
| gemma-q4 solo | 34.900 | 34.661 | OK | |
|
| 30 |
+
| gemma-q4 dflash-f16 x | 2.240 | 2.09 (med 2.19) | OK | mean 6.8% off, median 2.4% off |
|
| 31 |
+
| gemma-q4 mtp x | 2.500 | 2.39 (med 2.47) | OK | mean 4.4% off, median 1.0% off |
|
| 32 |
+
|
| 33 |
+
**24/24 checks pass.**
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-f16,math,1,60,0.8875,0.8965,0.8731,0.9004
|
| 3 |
+
gemma-q4-dflash-f16,math,2,60,0.7715,0.7805,0.7496,0.7916
|
| 4 |
+
gemma-q4-dflash-f16,math,3,60,0.6352,0.6340,0.6085,0.6602
|
| 5 |
+
gemma-q4-dflash-f16,code,1,60,0.8467,0.8600,0.8262,0.8678
|
| 6 |
+
gemma-q4-dflash-f16,code,2,60,0.7134,0.7145,0.6828,0.7452
|
| 7 |
+
gemma-q4-dflash-f16,code,3,60,0.5915,0.5775,0.5538,0.6293
|
| 8 |
+
gemma-q4-dflash-f16,chat,1,55,0.6516,0.6730,0.5918,0.7056
|
| 9 |
+
gemma-q4-dflash-f16,chat,2,55,0.4742,0.4350,0.4116,0.5377
|
| 10 |
+
gemma-q4-dflash-f16,chat,3,55,0.3220,0.2410,0.2602,0.3879
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-q4,math,1,60,0.8841,0.8940,0.8703,0.8966
|
| 3 |
+
gemma-q4-dflash-q4,math,2,60,0.7655,0.7855,0.7416,0.7874
|
| 4 |
+
gemma-q4-dflash-q4,math,3,60,0.6332,0.6340,0.6061,0.6592
|
| 5 |
+
gemma-q4-dflash-q4,code,1,60,0.8460,0.8535,0.8244,0.8675
|
| 6 |
+
gemma-q4-dflash-q4,code,2,60,0.7023,0.6975,0.6701,0.7339
|
| 7 |
+
gemma-q4-dflash-q4,code,3,60,0.5776,0.5865,0.5409,0.6155
|
| 8 |
+
gemma-q4-dflash-q4,chat,1,55,0.6411,0.6920,0.5845,0.6937
|
| 9 |
+
gemma-q4-dflash-q4,chat,2,55,0.4609,0.4530,0.4002,0.5217
|
| 10 |
+
gemma-q4-dflash-q4,chat,3,55,0.3073,0.2500,0.2477,0.3702
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-q8,math,1,60,0.8858,0.8965,0.8707,0.8988
|
| 3 |
+
gemma-q4-dflash-q8,math,2,60,0.7701,0.7805,0.7481,0.7899
|
| 4 |
+
gemma-q4-dflash-q8,math,3,60,0.6345,0.6365,0.6085,0.6599
|
| 5 |
+
gemma-q4-dflash-q8,code,1,60,0.8447,0.8590,0.8248,0.8653
|
| 6 |
+
gemma-q4-dflash-q8,code,2,60,0.7095,0.7055,0.6789,0.7401
|
| 7 |
+
gemma-q4-dflash-q8,code,3,60,0.5906,0.5775,0.5531,0.6284
|
| 8 |
+
gemma-q4-dflash-q8,chat,1,55,0.6511,0.6700,0.5908,0.7066
|
| 9 |
+
gemma-q4-dflash-q8,chat,2,55,0.4744,0.4390,0.4124,0.5378
|
| 10 |
+
gemma-q4-dflash-q8,chat,3,55,0.3221,0.2390,0.2609,0.3876
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-mtp.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-mtp,math,1,60,0.9051,0.9120,0.8935,0.9166
|
| 3 |
+
gemma-q4-mtp,math,2,60,0.8125,0.8160,0.7920,0.8327
|
| 4 |
+
gemma-q4-mtp,math,3,60,0.7374,0.7520,0.7137,0.7598
|
| 5 |
+
gemma-q4-mtp,code,1,60,0.8863,0.8960,0.8700,0.9020
|
| 6 |
+
gemma-q4-mtp,code,2,60,0.7817,0.7775,0.7572,0.8054
|
| 7 |
+
gemma-q4-mtp,code,3,60,0.6927,0.6885,0.6627,0.7231
|
| 8 |
+
gemma-q4-mtp,chat,1,55,0.7378,0.7860,0.6822,0.7893
|
| 9 |
+
gemma-q4-mtp,chat,2,55,0.5864,0.5740,0.5274,0.6486
|
| 10 |
+
gemma-q4-mtp,chat,3,55,0.4803,0.4360,0.4184,0.5485
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q5-dflash-f16,math,1,60,0.8929,0.9050,0.8811,0.9041
|
| 3 |
+
gemma-q5-dflash-f16,math,2,60,0.7866,0.7970,0.7673,0.8056
|
| 4 |
+
gemma-q5-dflash-f16,math,3,60,0.6725,0.6780,0.6515,0.6936
|
| 5 |
+
gemma-q5-dflash-f16,code,1,58,0.8493,0.8710,0.8153,0.8769
|
| 6 |
+
gemma-q5-dflash-f16,code,2,58,0.7110,0.7265,0.6739,0.7490
|
| 7 |
+
gemma-q5-dflash-f16,code,3,58,0.5873,0.5820,0.5447,0.6348
|
| 8 |
+
gemma-q5-dflash-f16,chat,1,56,0.6458,0.6975,0.5908,0.6954
|
| 9 |
+
gemma-q5-dflash-f16,chat,2,56,0.4307,0.4210,0.3778,0.4835
|
| 10 |
+
gemma-q5-dflash-f16,chat,3,56,0.2739,0.2405,0.2242,0.3253
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q5-mtp.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q5-mtp,math,1,60,0.9158,0.9170,0.9052,0.9254
|
| 3 |
+
gemma-q5-mtp,math,2,60,0.8320,0.8380,0.8127,0.8498
|
| 4 |
+
gemma-q5-mtp,math,3,60,0.7506,0.7570,0.7292,0.7723
|
| 5 |
+
gemma-q5-mtp,code,1,58,0.8731,0.8885,0.8504,0.8939
|
| 6 |
+
gemma-q5-mtp,code,2,58,0.7701,0.7775,0.7439,0.7976
|
| 7 |
+
gemma-q5-mtp,code,3,58,0.6722,0.6670,0.6396,0.7067
|
| 8 |
+
gemma-q5-mtp,chat,1,56,0.7448,0.7380,0.7027,0.7800
|
| 9 |
+
gemma-q5-mtp,chat,2,56,0.5743,0.5720,0.5256,0.6192
|
| 10 |
+
gemma-q5-mtp,chat,3,56,0.4511,0.4480,0.3983,0.5046
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q8-dflash-f16,math,1,60,0.8919,0.9045,0.8784,0.9040
|
| 3 |
+
gemma-q8-dflash-f16,math,2,60,0.7721,0.7930,0.7505,0.7939
|
| 4 |
+
gemma-q8-dflash-f16,math,3,60,0.6460,0.6535,0.6191,0.6729
|
| 5 |
+
gemma-q8-dflash-f16,code,1,58,0.8427,0.8595,0.8168,0.8670
|
| 6 |
+
gemma-q8-dflash-f16,code,2,58,0.7027,0.6790,0.6676,0.7374
|
| 7 |
+
gemma-q8-dflash-f16,code,3,58,0.5889,0.5605,0.5460,0.6324
|
| 8 |
+
gemma-q8-dflash-f16,chat,1,56,0.6705,0.6720,0.6234,0.7139
|
| 9 |
+
gemma-q8-dflash-f16,chat,2,56,0.4360,0.4165,0.3837,0.4903
|
| 10 |
+
gemma-q8-dflash-f16,chat,3,56,0.2905,0.2595,0.2419,0.3434
|
experiments/analysis/curves/acc_by_pos_curves-gemma-q8-mtp.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q8-mtp,math,1,60,0.9139,0.9285,0.9010,0.9263
|
| 3 |
+
gemma-q8-mtp,math,2,60,0.8342,0.8480,0.8141,0.8543
|
| 4 |
+
gemma-q8-mtp,math,3,60,0.7476,0.7625,0.7243,0.7714
|
| 5 |
+
gemma-q8-mtp,code,1,58,0.8692,0.8655,0.8525,0.8869
|
| 6 |
+
gemma-q8-mtp,code,2,58,0.7647,0.7455,0.7410,0.7911
|
| 7 |
+
gemma-q8-mtp,code,3,58,0.6648,0.6515,0.6358,0.6977
|
| 8 |
+
gemma-q8-mtp,chat,1,56,0.7583,0.7555,0.7149,0.7956
|
| 9 |
+
gemma-q8-mtp,chat,2,56,0.5967,0.5870,0.5498,0.6417
|
| 10 |
+
gemma-q8-mtp,chat,3,56,0.4819,0.4470,0.4278,0.5349
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dflash.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dflash-f16,math,1,60,0.0200,0.0160,0.0160,0.0238
|
| 3 |
+
qwen-q4-dflash-f16,math,2,60,0.0048,0.0040,0.0032,0.0069
|
| 4 |
+
qwen-q4-dflash-f16,math,3,60,0.0018,0.0000,0.0011,0.0025
|
| 5 |
+
qwen-q4-dflash-f16,code,1,60,0.0215,0.0200,0.0182,0.0248
|
| 6 |
+
qwen-q4-dflash-f16,code,2,60,0.0043,0.0040,0.0033,0.0054
|
| 7 |
+
qwen-q4-dflash-f16,code,3,60,0.0018,0.0000,0.0011,0.0025
|
| 8 |
+
qwen-q4-dflash-f16,chat,1,58,0.0284,0.0205,0.0233,0.0338
|
| 9 |
+
qwen-q4-dflash-f16,chat,2,58,0.0042,0.0040,0.0032,0.0053
|
| 10 |
+
qwen-q4-dflash-f16,chat,3,58,0.0016,0.0000,0.0010,0.0022
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p0,math,1,60,0.8347,0.8510,0.8121,0.8570
|
| 3 |
+
qwen-q4-dspark-p0,math,2,60,0.6921,0.6975,0.6581,0.7273
|
| 4 |
+
qwen-q4-dspark-p0,math,3,60,0.5783,0.5645,0.5370,0.6213
|
| 5 |
+
qwen-q4-dspark-p0,code,1,60,0.7783,0.7705,0.7597,0.7969
|
| 6 |
+
qwen-q4-dspark-p0,code,2,60,0.6005,0.5930,0.5761,0.6254
|
| 7 |
+
qwen-q4-dspark-p0,code,3,60,0.4507,0.4320,0.4226,0.4816
|
| 8 |
+
qwen-q4-dspark-p0,chat,1,58,0.6920,0.7335,0.6444,0.7339
|
| 9 |
+
qwen-q4-dspark-p0,chat,2,58,0.4769,0.4975,0.4254,0.5239
|
| 10 |
+
qwen-q4-dspark-p0,chat,3,58,0.3412,0.3415,0.2928,0.3871
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p2,math,1,60,0.8345,0.8510,0.8120,0.8567
|
| 3 |
+
qwen-q4-dspark-p2,math,2,60,0.6918,0.6975,0.6576,0.7270
|
| 4 |
+
qwen-q4-dspark-p2,math,3,60,0.5783,0.5645,0.5370,0.6215
|
| 5 |
+
qwen-q4-dspark-p2,code,1,60,0.7778,0.7705,0.7592,0.7964
|
| 6 |
+
qwen-q4-dspark-p2,code,2,60,0.6003,0.5910,0.5758,0.6250
|
| 7 |
+
qwen-q4-dspark-p2,code,3,60,0.4508,0.4320,0.4225,0.4818
|
| 8 |
+
qwen-q4-dspark-p2,chat,1,58,0.6981,0.7405,0.6547,0.7378
|
| 9 |
+
qwen-q4-dspark-p2,chat,2,58,0.4825,0.5025,0.4326,0.5286
|
| 10 |
+
qwen-q4-dspark-p2,chat,3,58,0.3469,0.3605,0.2989,0.3935
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p4,math,1,60,0.8353,0.8520,0.8121,0.8580
|
| 3 |
+
qwen-q4-dspark-p4,math,2,60,0.6887,0.7000,0.6534,0.7253
|
| 4 |
+
qwen-q4-dspark-p4,math,3,60,0.5757,0.5560,0.5340,0.6201
|
| 5 |
+
qwen-q4-dspark-p4,code,1,60,0.7806,0.7890,0.7626,0.7979
|
| 6 |
+
qwen-q4-dspark-p4,code,2,60,0.6029,0.5945,0.5797,0.6280
|
| 7 |
+
qwen-q4-dspark-p4,code,3,60,0.4496,0.4325,0.4219,0.4785
|
| 8 |
+
qwen-q4-dspark-p4,chat,1,58,0.7250,0.7370,0.6935,0.7549
|
| 9 |
+
qwen-q4-dspark-p4,chat,2,58,0.4815,0.4800,0.4334,0.5258
|
| 10 |
+
qwen-q4-dspark-p4,chat,3,58,0.3297,0.3185,0.2819,0.3738
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p6,math,1,60,0.8438,0.8585,0.8235,0.8640
|
| 3 |
+
qwen-q4-dspark-p6,math,2,60,0.6839,0.6960,0.6493,0.7189
|
| 4 |
+
qwen-q4-dspark-p6,math,3,60,0.5639,0.5430,0.5213,0.6084
|
| 5 |
+
qwen-q4-dspark-p6,code,1,60,0.7965,0.7920,0.7783,0.8139
|
| 6 |
+
qwen-q4-dspark-p6,code,2,60,0.5830,0.5850,0.5583,0.6082
|
| 7 |
+
qwen-q4-dspark-p6,code,3,60,0.4117,0.4155,0.3843,0.4407
|
| 8 |
+
qwen-q4-dspark-p6,chat,1,58,0.7730,0.7805,0.7448,0.7983
|
| 9 |
+
qwen-q4-dspark-p6,chat,2,58,0.4486,0.4460,0.4033,0.4906
|
| 10 |
+
qwen-q4-dspark-p6,chat,3,58,0.2883,0.2700,0.2423,0.3329
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-eagle3.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-eagle3,math,1,60,0.6590,0.6410,0.6399,0.6790
|
| 3 |
+
qwen-q4-eagle3,math,2,60,0.4375,0.4340,0.4135,0.4620
|
| 4 |
+
qwen-q4-eagle3,math,3,60,0.2376,0.2375,0.2184,0.2586
|
| 5 |
+
qwen-q4-eagle3,code,1,60,0.6724,0.6715,0.6527,0.6925
|
| 6 |
+
qwen-q4-eagle3,code,2,60,0.4320,0.4195,0.4065,0.4582
|
| 7 |
+
qwen-q4-eagle3,code,3,60,0.2209,0.1965,0.1932,0.2509
|
| 8 |
+
qwen-q4-eagle3,chat,1,58,0.5734,0.6125,0.5174,0.6248
|
| 9 |
+
qwen-q4-eagle3,chat,2,58,0.3631,0.3525,0.3077,0.4165
|
| 10 |
+
qwen-q4-eagle3,chat,3,58,0.2106,0.1650,0.1654,0.2585
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-vanilla17b,math,1,60,0.8725,0.8700,0.8594,0.8849
|
| 3 |
+
qwen-q4-vanilla17b,math,2,60,0.7706,0.7760,0.7498,0.7904
|
| 4 |
+
qwen-q4-vanilla17b,math,3,60,0.6855,0.6905,0.6592,0.7101
|
| 5 |
+
qwen-q4-vanilla17b,code,1,60,0.8352,0.8290,0.8234,0.8467
|
| 6 |
+
qwen-q4-vanilla17b,code,2,60,0.7095,0.7055,0.6907,0.7287
|
| 7 |
+
qwen-q4-vanilla17b,code,3,60,0.6087,0.6065,0.5858,0.6330
|
| 8 |
+
qwen-q4-vanilla17b,chat,1,58,0.7829,0.7745,0.7623,0.8032
|
| 9 |
+
qwen-q4-vanilla17b,chat,2,58,0.6254,0.6080,0.5945,0.6550
|
| 10 |
+
qwen-q4-vanilla17b,chat,3,58,0.5064,0.4880,0.4686,0.5435
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dflash.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q5-dflash-f16,math,1,60,0.0197,0.0160,0.0161,0.0233
|
| 3 |
+
qwen-q5-dflash-f16,math,2,60,0.0045,0.0040,0.0032,0.0059
|
| 4 |
+
qwen-q5-dflash-f16,math,3,60,0.0022,0.0000,0.0015,0.0031
|
| 5 |
+
qwen-q5-dflash-f16,code,1,60,0.0203,0.0200,0.0174,0.0233
|
| 6 |
+
qwen-q5-dflash-f16,code,2,60,0.0048,0.0040,0.0034,0.0063
|
| 7 |
+
qwen-q5-dflash-f16,code,3,60,0.0019,0.0000,0.0012,0.0026
|
| 8 |
+
qwen-q5-dflash-f16,chat,1,58,0.0264,0.0240,0.0217,0.0318
|
| 9 |
+
qwen-q5-dflash-f16,chat,2,58,0.0035,0.0040,0.0026,0.0044
|
| 10 |
+
qwen-q5-dflash-f16,chat,3,58,0.0013,0.0000,0.0008,0.0019
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q5-dspark-p0,math,1,60,0.8289,0.8320,0.8069,0.8512
|
| 3 |
+
qwen-q5-dspark-p0,math,2,60,0.6885,0.6895,0.6549,0.7223
|
| 4 |
+
qwen-q5-dspark-p0,math,3,60,0.5804,0.5835,0.5396,0.6222
|
| 5 |
+
qwen-q5-dspark-p0,code,1,60,0.7709,0.7685,0.7531,0.7896
|
| 6 |
+
qwen-q5-dspark-p0,code,2,60,0.5833,0.5815,0.5557,0.6125
|
| 7 |
+
qwen-q5-dspark-p0,code,3,60,0.4356,0.4285,0.4027,0.4701
|
| 8 |
+
qwen-q5-dspark-p0,chat,1,58,0.7017,0.7355,0.6575,0.7423
|
| 9 |
+
qwen-q5-dspark-p0,chat,2,58,0.4864,0.4925,0.4364,0.5348
|
| 10 |
+
qwen-q5-dspark-p0,chat,3,58,0.3534,0.3285,0.3056,0.4057
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-eagle3.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q5-eagle3,math,1,60,0.6618,0.6515,0.6443,0.6798
|
| 3 |
+
qwen-q5-eagle3,math,2,60,0.4344,0.4290,0.4129,0.4560
|
| 4 |
+
qwen-q5-eagle3,math,3,60,0.2333,0.2145,0.2162,0.2506
|
| 5 |
+
qwen-q5-eagle3,code,1,60,0.6694,0.6685,0.6487,0.6908
|
| 6 |
+
qwen-q5-eagle3,code,2,60,0.4253,0.4245,0.3992,0.4533
|
| 7 |
+
qwen-q5-eagle3,code,3,60,0.2182,0.1960,0.1913,0.2485
|
| 8 |
+
qwen-q5-eagle3,chat,1,58,0.5931,0.6385,0.5418,0.6432
|
| 9 |
+
qwen-q5-eagle3,chat,2,58,0.3617,0.3620,0.3138,0.4103
|
| 10 |
+
qwen-q5-eagle3,chat,3,58,0.2002,0.1630,0.1611,0.2424
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q8-dspark-p0,math,1,60,0.8264,0.8365,0.8037,0.8494
|
| 3 |
+
qwen-q8-dspark-p0,math,2,60,0.6872,0.6920,0.6529,0.7225
|
| 4 |
+
qwen-q8-dspark-p0,math,3,60,0.5736,0.5815,0.5309,0.6169
|
| 5 |
+
qwen-q8-dspark-p0,code,1,60,0.7736,0.7720,0.7569,0.7909
|
| 6 |
+
qwen-q8-dspark-p0,code,2,60,0.5855,0.5790,0.5615,0.6116
|
| 7 |
+
qwen-q8-dspark-p0,code,3,60,0.4410,0.4405,0.4163,0.4681
|
| 8 |
+
qwen-q8-dspark-p0,chat,1,58,0.6960,0.7210,0.6522,0.7347
|
| 9 |
+
qwen-q8-dspark-p0,chat,2,58,0.4866,0.4830,0.4410,0.5306
|
| 10 |
+
qwen-q8-dspark-p0,chat,3,58,0.3454,0.3235,0.3001,0.3922
|
experiments/analysis/curves/acc_by_pos_curves-qwen-q8-eagle3.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q8-eagle3,math,1,60,0.6504,0.6445,0.6335,0.6669
|
| 3 |
+
qwen-q8-eagle3,math,2,60,0.4285,0.4180,0.4102,0.4492
|
| 4 |
+
qwen-q8-eagle3,math,3,60,0.2302,0.2345,0.2142,0.2476
|
| 5 |
+
qwen-q8-eagle3,code,1,60,0.6651,0.6535,0.6473,0.6835
|
| 6 |
+
qwen-q8-eagle3,code,2,60,0.4249,0.4165,0.4029,0.4487
|
| 7 |
+
qwen-q8-eagle3,code,3,60,0.2141,0.1890,0.1900,0.2404
|
| 8 |
+
qwen-q8-eagle3,chat,1,58,0.5834,0.6370,0.5311,0.6310
|
| 9 |
+
qwen-q8-eagle3,chat,2,58,0.3571,0.3720,0.3112,0.4011
|
| 10 |
+
qwen-q8-eagle3,chat,3,58,0.1905,0.1800,0.1568,0.2269
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-f16,math,1,60,0.8581,0.8600,0.8396,0.8743
|
| 3 |
+
gemma-q4-dflash-f16,math,2,60,0.7291,0.7255,0.7029,0.7541
|
| 4 |
+
gemma-q4-dflash-f16,math,3,60,0.5859,0.5935,0.5582,0.6130
|
| 5 |
+
gemma-q4-dflash-f16,math,4,60,0.4880,0.4710,0.4562,0.5191
|
| 6 |
+
gemma-q4-dflash-f16,math,5,60,0.3957,0.3750,0.3630,0.4270
|
| 7 |
+
gemma-q4-dflash-f16,math,6,60,0.3183,0.3030,0.2860,0.3491
|
| 8 |
+
gemma-q4-dflash-f16,math,7,60,0.2572,0.2255,0.2282,0.2844
|
| 9 |
+
gemma-q4-dflash-f16,math,8,60,0.2047,0.1840,0.1778,0.2304
|
| 10 |
+
gemma-q4-dflash-f16,math,9,60,0.1634,0.1615,0.1402,0.1858
|
| 11 |
+
gemma-q4-dflash-f16,math,10,60,0.1190,0.1020,0.0994,0.1383
|
| 12 |
+
gemma-q4-dflash-f16,code,1,60,0.7976,0.8050,0.7639,0.8275
|
| 13 |
+
gemma-q4-dflash-f16,code,2,60,0.6295,0.6485,0.5894,0.6649
|
| 14 |
+
gemma-q4-dflash-f16,code,3,60,0.4984,0.4920,0.4579,0.5403
|
| 15 |
+
gemma-q4-dflash-f16,code,4,60,0.3998,0.3985,0.3604,0.4414
|
| 16 |
+
gemma-q4-dflash-f16,code,5,60,0.3186,0.2845,0.2803,0.3592
|
| 17 |
+
gemma-q4-dflash-f16,code,6,60,0.2594,0.2250,0.2256,0.2962
|
| 18 |
+
gemma-q4-dflash-f16,code,7,60,0.2062,0.1760,0.1759,0.2401
|
| 19 |
+
gemma-q4-dflash-f16,code,8,60,0.1638,0.1355,0.1355,0.1968
|
| 20 |
+
gemma-q4-dflash-f16,code,9,60,0.1324,0.0935,0.1053,0.1640
|
| 21 |
+
gemma-q4-dflash-f16,code,10,60,0.1047,0.0765,0.0809,0.1346
|
| 22 |
+
gemma-q4-dflash-f16,chat,1,55,0.6401,0.6670,0.5908,0.6840
|
| 23 |
+
gemma-q4-dflash-f16,chat,2,55,0.4230,0.4220,0.3681,0.4749
|
| 24 |
+
gemma-q4-dflash-f16,chat,3,55,0.2565,0.2060,0.2103,0.3053
|
| 25 |
+
gemma-q4-dflash-f16,chat,4,55,0.1807,0.1370,0.1361,0.2264
|
| 26 |
+
gemma-q4-dflash-f16,chat,5,55,0.1324,0.0820,0.0922,0.1771
|
| 27 |
+
gemma-q4-dflash-f16,chat,6,55,0.0946,0.0410,0.0585,0.1332
|
| 28 |
+
gemma-q4-dflash-f16,chat,7,55,0.0712,0.0200,0.0384,0.1063
|
| 29 |
+
gemma-q4-dflash-f16,chat,8,55,0.0589,0.0110,0.0292,0.0933
|
| 30 |
+
gemma-q4-dflash-f16,chat,9,55,0.0447,0.0090,0.0187,0.0754
|
| 31 |
+
gemma-q4-dflash-f16,chat,10,55,0.0355,0.0000,0.0134,0.0626
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-f16,math,1,60,0.8648,0.8805,0.8466,0.8817
|
| 3 |
+
gemma-q4-dflash-f16,math,2,60,0.7476,0.7540,0.7238,0.7708
|
| 4 |
+
gemma-q4-dflash-f16,math,3,60,0.6149,0.6180,0.5844,0.6425
|
| 5 |
+
gemma-q4-dflash-f16,math,4,60,0.5182,0.5040,0.4846,0.5506
|
| 6 |
+
gemma-q4-dflash-f16,math,5,60,0.4348,0.4370,0.4013,0.4672
|
| 7 |
+
gemma-q4-dflash-f16,code,1,60,0.8291,0.8310,0.8066,0.8516
|
| 8 |
+
gemma-q4-dflash-f16,code,2,60,0.6822,0.6800,0.6523,0.7136
|
| 9 |
+
gemma-q4-dflash-f16,code,3,60,0.5617,0.5665,0.5240,0.5997
|
| 10 |
+
gemma-q4-dflash-f16,code,4,60,0.4657,0.4620,0.4247,0.5074
|
| 11 |
+
gemma-q4-dflash-f16,code,5,60,0.3904,0.3770,0.3486,0.4327
|
| 12 |
+
gemma-q4-dflash-f16,chat,1,55,0.6564,0.6800,0.6046,0.7042
|
| 13 |
+
gemma-q4-dflash-f16,chat,2,55,0.4466,0.4130,0.3915,0.5006
|
| 14 |
+
gemma-q4-dflash-f16,chat,3,55,0.2992,0.2500,0.2453,0.3531
|
| 15 |
+
gemma-q4-dflash-f16,chat,4,55,0.2237,0.1490,0.1725,0.2766
|
| 16 |
+
gemma-q4-dflash-f16,chat,5,55,0.1691,0.1060,0.1185,0.2226
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-dflash-f16,math,1,60,0.8579,0.8670,0.8395,0.8751
|
| 3 |
+
gemma-q4-dflash-f16,math,2,60,0.7314,0.7405,0.7069,0.7555
|
| 4 |
+
gemma-q4-dflash-f16,math,3,60,0.5956,0.6050,0.5641,0.6247
|
| 5 |
+
gemma-q4-dflash-f16,math,4,60,0.4932,0.4820,0.4577,0.5276
|
| 6 |
+
gemma-q4-dflash-f16,math,5,60,0.4169,0.3910,0.3822,0.4509
|
| 7 |
+
gemma-q4-dflash-f16,math,6,60,0.3513,0.3330,0.3177,0.3852
|
| 8 |
+
gemma-q4-dflash-f16,math,7,60,0.2898,0.2665,0.2592,0.3210
|
| 9 |
+
gemma-q4-dflash-f16,code,1,60,0.8238,0.8230,0.7995,0.8470
|
| 10 |
+
gemma-q4-dflash-f16,code,2,60,0.6661,0.6670,0.6331,0.6983
|
| 11 |
+
gemma-q4-dflash-f16,code,3,60,0.5352,0.5340,0.4997,0.5731
|
| 12 |
+
gemma-q4-dflash-f16,code,4,60,0.4426,0.4410,0.4046,0.4808
|
| 13 |
+
gemma-q4-dflash-f16,code,5,60,0.3694,0.3695,0.3308,0.4081
|
| 14 |
+
gemma-q4-dflash-f16,code,6,60,0.3013,0.2845,0.2641,0.3397
|
| 15 |
+
gemma-q4-dflash-f16,code,7,60,0.2465,0.2235,0.2120,0.2846
|
| 16 |
+
gemma-q4-dflash-f16,chat,1,55,0.6426,0.6580,0.5935,0.6863
|
| 17 |
+
gemma-q4-dflash-f16,chat,2,55,0.4293,0.4090,0.3773,0.4805
|
| 18 |
+
gemma-q4-dflash-f16,chat,3,55,0.2806,0.2420,0.2307,0.3323
|
| 19 |
+
gemma-q4-dflash-f16,chat,4,55,0.2043,0.1510,0.1562,0.2537
|
| 20 |
+
gemma-q4-dflash-f16,chat,5,55,0.1530,0.0870,0.1073,0.2019
|
| 21 |
+
gemma-q4-dflash-f16,chat,6,55,0.1202,0.0510,0.0773,0.1675
|
| 22 |
+
gemma-q4-dflash-f16,chat,7,55,0.0942,0.0280,0.0528,0.1403
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-mtp,math,1,60,0.8837,0.8905,0.8682,0.8992
|
| 3 |
+
gemma-q4-mtp,math,2,60,0.7941,0.8000,0.7706,0.8170
|
| 4 |
+
gemma-q4-mtp,math,3,60,0.7048,0.7195,0.6784,0.7300
|
| 5 |
+
gemma-q4-mtp,math,4,60,0.6123,0.6385,0.5747,0.6443
|
| 6 |
+
gemma-q4-mtp,math,5,60,0.5342,0.5490,0.4978,0.5670
|
| 7 |
+
gemma-q4-mtp,math,6,60,0.4595,0.4630,0.4243,0.4929
|
| 8 |
+
gemma-q4-mtp,math,7,60,0.3834,0.3735,0.3502,0.4155
|
| 9 |
+
gemma-q4-mtp,math,8,60,0.3149,0.3115,0.2849,0.3450
|
| 10 |
+
gemma-q4-mtp,math,9,60,0.2509,0.2415,0.2232,0.2811
|
| 11 |
+
gemma-q4-mtp,math,10,60,0.2024,0.2000,0.1794,0.2282
|
| 12 |
+
gemma-q4-mtp,code,1,60,0.8295,0.8435,0.8010,0.8528
|
| 13 |
+
gemma-q4-mtp,code,2,60,0.7212,0.7320,0.6897,0.7507
|
| 14 |
+
gemma-q4-mtp,code,3,60,0.6143,0.6040,0.5809,0.6463
|
| 15 |
+
gemma-q4-mtp,code,4,60,0.5351,0.5230,0.5003,0.5712
|
| 16 |
+
gemma-q4-mtp,code,5,60,0.4681,0.4495,0.4327,0.5033
|
| 17 |
+
gemma-q4-mtp,code,6,60,0.3917,0.3770,0.3582,0.4263
|
| 18 |
+
gemma-q4-mtp,code,7,60,0.3331,0.3090,0.2998,0.3674
|
| 19 |
+
gemma-q4-mtp,code,8,60,0.2793,0.2500,0.2477,0.3137
|
| 20 |
+
gemma-q4-mtp,code,9,60,0.2336,0.2140,0.2039,0.2662
|
| 21 |
+
gemma-q4-mtp,code,10,60,0.1817,0.1590,0.1529,0.2141
|
| 22 |
+
gemma-q4-mtp,chat,1,55,0.7338,0.7470,0.6891,0.7732
|
| 23 |
+
gemma-q4-mtp,chat,2,55,0.5758,0.5690,0.5250,0.6269
|
| 24 |
+
gemma-q4-mtp,chat,3,55,0.4411,0.3880,0.3849,0.5014
|
| 25 |
+
gemma-q4-mtp,chat,4,55,0.3483,0.2770,0.2882,0.4102
|
| 26 |
+
gemma-q4-mtp,chat,5,55,0.2700,0.1920,0.2119,0.3302
|
| 27 |
+
gemma-q4-mtp,chat,6,55,0.2250,0.1390,0.1672,0.2859
|
| 28 |
+
gemma-q4-mtp,chat,7,55,0.1913,0.0920,0.1341,0.2554
|
| 29 |
+
gemma-q4-mtp,chat,8,55,0.1640,0.0690,0.1064,0.2285
|
| 30 |
+
gemma-q4-mtp,chat,9,55,0.1394,0.0430,0.0820,0.2036
|
| 31 |
+
gemma-q4-mtp,chat,10,55,0.1244,0.0280,0.0660,0.1884
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-mtp,math,1,60,0.8950,0.8975,0.8799,0.9086
|
| 3 |
+
gemma-q4-mtp,math,2,60,0.8062,0.8085,0.7859,0.8247
|
| 4 |
+
gemma-q4-mtp,math,3,60,0.7245,0.7430,0.6978,0.7487
|
| 5 |
+
gemma-q4-mtp,math,4,60,0.6364,0.6530,0.6067,0.6637
|
| 6 |
+
gemma-q4-mtp,math,5,60,0.5608,0.5600,0.5306,0.5901
|
| 7 |
+
gemma-q4-mtp,code,1,60,0.8678,0.8850,0.8489,0.8859
|
| 8 |
+
gemma-q4-mtp,code,2,60,0.7681,0.7740,0.7418,0.7935
|
| 9 |
+
gemma-q4-mtp,code,3,60,0.6668,0.6610,0.6358,0.6977
|
| 10 |
+
gemma-q4-mtp,code,4,60,0.5806,0.5680,0.5440,0.6167
|
| 11 |
+
gemma-q4-mtp,code,5,60,0.5110,0.5000,0.4725,0.5504
|
| 12 |
+
gemma-q4-mtp,chat,1,55,0.7516,0.7570,0.7078,0.7899
|
| 13 |
+
gemma-q4-mtp,chat,2,55,0.5933,0.5850,0.5418,0.6487
|
| 14 |
+
gemma-q4-mtp,chat,3,55,0.4816,0.4120,0.4221,0.5442
|
| 15 |
+
gemma-q4-mtp,chat,4,55,0.3951,0.3250,0.3306,0.4627
|
| 16 |
+
gemma-q4-mtp,chat,5,55,0.3237,0.2190,0.2579,0.3945
|
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
gemma-q4-mtp,math,1,60,0.8807,0.8830,0.8625,0.8965
|
| 3 |
+
gemma-q4-mtp,math,2,60,0.7945,0.8025,0.7706,0.8170
|
| 4 |
+
gemma-q4-mtp,math,3,60,0.7120,0.7085,0.6852,0.7385
|
| 5 |
+
gemma-q4-mtp,math,4,60,0.6326,0.6380,0.6015,0.6633
|
| 6 |
+
gemma-q4-mtp,math,5,60,0.5558,0.5530,0.5223,0.5895
|
| 7 |
+
gemma-q4-mtp,math,6,60,0.4760,0.4825,0.4397,0.5133
|
| 8 |
+
gemma-q4-mtp,math,7,60,0.4050,0.3900,0.3713,0.4385
|
| 9 |
+
gemma-q4-mtp,code,1,60,0.8569,0.8740,0.8370,0.8764
|
| 10 |
+
gemma-q4-mtp,code,2,60,0.7463,0.7630,0.7187,0.7722
|
| 11 |
+
gemma-q4-mtp,code,3,60,0.6522,0.6540,0.6200,0.6846
|
| 12 |
+
gemma-q4-mtp,code,4,60,0.5666,0.5680,0.5307,0.6020
|
| 13 |
+
gemma-q4-mtp,code,5,60,0.4945,0.4755,0.4555,0.5327
|
| 14 |
+
gemma-q4-mtp,code,6,60,0.4205,0.3970,0.3808,0.4608
|
| 15 |
+
gemma-q4-mtp,code,7,60,0.3630,0.3365,0.3227,0.4041
|
| 16 |
+
gemma-q4-mtp,chat,1,55,0.7442,0.7500,0.7010,0.7816
|
| 17 |
+
gemma-q4-mtp,chat,2,55,0.5852,0.5610,0.5343,0.6413
|
| 18 |
+
gemma-q4-mtp,chat,3,55,0.4759,0.4110,0.4179,0.5385
|
| 19 |
+
gemma-q4-mtp,chat,4,55,0.3938,0.3010,0.3297,0.4655
|
| 20 |
+
gemma-q4-mtp,chat,5,55,0.3201,0.2030,0.2554,0.3891
|
| 21 |
+
gemma-q4-mtp,chat,6,55,0.2712,0.1590,0.2051,0.3428
|
| 22 |
+
gemma-q4-mtp,chat,7,55,0.2360,0.1130,0.1668,0.3091
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dflash-f16,math,1,60,0.0147,0.0100,0.0113,0.0186
|
| 3 |
+
qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0018,0.0056
|
| 4 |
+
qwen-q4-dflash-f16,math,3,60,0.0017,0.0000,0.0009,0.0025
|
| 5 |
+
qwen-q4-dflash-f16,math,4,60,0.0011,0.0000,0.0007,0.0017
|
| 6 |
+
qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
|
| 7 |
+
qwen-q4-dflash-f16,math,6,60,0.0003,0.0000,0.0001,0.0007
|
| 8 |
+
qwen-q4-dflash-f16,math,7,60,0.0000,0.0000,0.0000,0.0000
|
| 9 |
+
qwen-q4-dflash-f16,math,8,60,0.0000,0.0000,0.0000,0.0000
|
| 10 |
+
qwen-q4-dflash-f16,math,9,60,0.0000,0.0000,0.0000,0.0000
|
| 11 |
+
qwen-q4-dflash-f16,math,10,60,0.0000,0.0000,0.0000,0.0000
|
| 12 |
+
qwen-q4-dflash-f16,code,1,60,0.0138,0.0120,0.0112,0.0165
|
| 13 |
+
qwen-q4-dflash-f16,code,2,60,0.0039,0.0040,0.0028,0.0052
|
| 14 |
+
qwen-q4-dflash-f16,code,3,60,0.0015,0.0000,0.0009,0.0024
|
| 15 |
+
qwen-q4-dflash-f16,code,4,60,0.0006,0.0000,0.0002,0.0011
|
| 16 |
+
qwen-q4-dflash-f16,code,5,60,0.0003,0.0000,0.0001,0.0007
|
| 17 |
+
qwen-q4-dflash-f16,code,6,60,0.0002,0.0000,0.0000,0.0005
|
| 18 |
+
qwen-q4-dflash-f16,code,7,60,0.0000,0.0000,0.0000,0.0000
|
| 19 |
+
qwen-q4-dflash-f16,code,8,60,0.0000,0.0000,0.0000,0.0000
|
| 20 |
+
qwen-q4-dflash-f16,code,9,60,0.0000,0.0000,0.0000,0.0000
|
| 21 |
+
qwen-q4-dflash-f16,code,10,60,0.0000,0.0000,0.0000,0.0000
|
| 22 |
+
qwen-q4-dflash-f16,chat,1,58,0.0214,0.0160,0.0171,0.0264
|
| 23 |
+
qwen-q4-dflash-f16,chat,2,58,0.0046,0.0040,0.0033,0.0060
|
| 24 |
+
qwen-q4-dflash-f16,chat,3,58,0.0019,0.0000,0.0011,0.0027
|
| 25 |
+
qwen-q4-dflash-f16,chat,4,58,0.0010,0.0000,0.0004,0.0017
|
| 26 |
+
qwen-q4-dflash-f16,chat,5,58,0.0005,0.0000,0.0001,0.0011
|
| 27 |
+
qwen-q4-dflash-f16,chat,6,58,0.0003,0.0000,0.0000,0.0007
|
| 28 |
+
qwen-q4-dflash-f16,chat,7,58,0.0000,0.0000,0.0000,0.0000
|
| 29 |
+
qwen-q4-dflash-f16,chat,8,58,0.0000,0.0000,0.0000,0.0000
|
| 30 |
+
qwen-q4-dflash-f16,chat,9,58,0.0000,0.0000,0.0000,0.0000
|
| 31 |
+
qwen-q4-dflash-f16,chat,10,58,0.0000,0.0000,0.0000,0.0000
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dflash-f16,math,1,60,0.0139,0.0120,0.0105,0.0178
|
| 3 |
+
qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0019,0.0056
|
| 4 |
+
qwen-q4-dflash-f16,math,3,60,0.0013,0.0000,0.0007,0.0019
|
| 5 |
+
qwen-q4-dflash-f16,math,4,60,0.0008,0.0000,0.0004,0.0012
|
| 6 |
+
qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
|
| 7 |
+
qwen-q4-dflash-f16,code,1,60,0.0126,0.0120,0.0101,0.0152
|
| 8 |
+
qwen-q4-dflash-f16,code,2,60,0.0030,0.0000,0.0020,0.0040
|
| 9 |
+
qwen-q4-dflash-f16,code,3,60,0.0013,0.0000,0.0007,0.0018
|
| 10 |
+
qwen-q4-dflash-f16,code,4,60,0.0005,0.0000,0.0002,0.0009
|
| 11 |
+
qwen-q4-dflash-f16,code,5,60,0.0002,0.0000,0.0000,0.0005
|
| 12 |
+
qwen-q4-dflash-f16,chat,1,58,0.0209,0.0160,0.0166,0.0257
|
| 13 |
+
qwen-q4-dflash-f16,chat,2,58,0.0036,0.0000,0.0024,0.0051
|
| 14 |
+
qwen-q4-dflash-f16,chat,3,58,0.0012,0.0000,0.0007,0.0018
|
| 15 |
+
qwen-q4-dflash-f16,chat,4,58,0.0006,0.0000,0.0003,0.0010
|
| 16 |
+
qwen-q4-dflash-f16,chat,5,58,0.0002,0.0000,0.0000,0.0005
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dflash-f16,math,1,60,0.0147,0.0100,0.0113,0.0186
|
| 3 |
+
qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0018,0.0056
|
| 4 |
+
qwen-q4-dflash-f16,math,3,60,0.0017,0.0000,0.0009,0.0025
|
| 5 |
+
qwen-q4-dflash-f16,math,4,60,0.0011,0.0000,0.0007,0.0017
|
| 6 |
+
qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
|
| 7 |
+
qwen-q4-dflash-f16,math,6,60,0.0003,0.0000,0.0001,0.0007
|
| 8 |
+
qwen-q4-dflash-f16,math,7,60,0.0000,0.0000,0.0000,0.0000
|
| 9 |
+
qwen-q4-dflash-f16,code,1,60,0.0138,0.0120,0.0112,0.0165
|
| 10 |
+
qwen-q4-dflash-f16,code,2,60,0.0039,0.0040,0.0028,0.0052
|
| 11 |
+
qwen-q4-dflash-f16,code,3,60,0.0015,0.0000,0.0009,0.0024
|
| 12 |
+
qwen-q4-dflash-f16,code,4,60,0.0006,0.0000,0.0002,0.0011
|
| 13 |
+
qwen-q4-dflash-f16,code,5,60,0.0003,0.0000,0.0001,0.0007
|
| 14 |
+
qwen-q4-dflash-f16,code,6,60,0.0002,0.0000,0.0000,0.0005
|
| 15 |
+
qwen-q4-dflash-f16,code,7,60,0.0000,0.0000,0.0000,0.0000
|
| 16 |
+
qwen-q4-dflash-f16,chat,1,58,0.0214,0.0160,0.0171,0.0264
|
| 17 |
+
qwen-q4-dflash-f16,chat,2,58,0.0046,0.0040,0.0033,0.0060
|
| 18 |
+
qwen-q4-dflash-f16,chat,3,58,0.0019,0.0000,0.0011,0.0027
|
| 19 |
+
qwen-q4-dflash-f16,chat,4,58,0.0010,0.0000,0.0004,0.0017
|
| 20 |
+
qwen-q4-dflash-f16,chat,5,58,0.0005,0.0000,0.0001,0.0011
|
| 21 |
+
qwen-q4-dflash-f16,chat,6,58,0.0003,0.0000,0.0000,0.0007
|
| 22 |
+
qwen-q4-dflash-f16,chat,7,58,0.0000,0.0000,0.0000,0.0000
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p0,math,1,60,0.8275,0.8235,0.8088,0.8465
|
| 3 |
+
qwen-q4-dspark-p0,math,2,60,0.6883,0.7065,0.6567,0.7224
|
| 4 |
+
qwen-q4-dspark-p0,math,3,60,0.5771,0.5700,0.5378,0.6177
|
| 5 |
+
qwen-q4-dspark-p0,math,4,60,0.4843,0.4585,0.4372,0.5337
|
| 6 |
+
qwen-q4-dspark-p0,math,5,60,0.4117,0.4000,0.3624,0.4656
|
| 7 |
+
qwen-q4-dspark-p0,math,6,60,0.3543,0.3390,0.3045,0.4083
|
| 8 |
+
qwen-q4-dspark-p0,math,7,60,0.3056,0.2950,0.2568,0.3597
|
| 9 |
+
qwen-q4-dspark-p0,math,8,60,0.0000,0.0000,0.0000,0.0000
|
| 10 |
+
qwen-q4-dspark-p0,math,9,60,0.0000,0.0000,0.0000,0.0000
|
| 11 |
+
qwen-q4-dspark-p0,math,10,60,0.0000,0.0000,0.0000,0.0000
|
| 12 |
+
qwen-q4-dspark-p0,code,1,60,0.7944,0.8080,0.7753,0.8127
|
| 13 |
+
qwen-q4-dspark-p0,code,2,60,0.6135,0.6125,0.5853,0.6421
|
| 14 |
+
qwen-q4-dspark-p0,code,3,60,0.4618,0.4355,0.4318,0.4931
|
| 15 |
+
qwen-q4-dspark-p0,code,4,60,0.3523,0.3255,0.3217,0.3855
|
| 16 |
+
qwen-q4-dspark-p0,code,5,60,0.2684,0.2355,0.2363,0.3034
|
| 17 |
+
qwen-q4-dspark-p0,code,6,60,0.1977,0.1725,0.1691,0.2283
|
| 18 |
+
qwen-q4-dspark-p0,code,7,60,0.1526,0.1190,0.1241,0.1840
|
| 19 |
+
qwen-q4-dspark-p0,code,8,60,0.0000,0.0000,0.0000,0.0000
|
| 20 |
+
qwen-q4-dspark-p0,code,9,60,0.0000,0.0000,0.0000,0.0000
|
| 21 |
+
qwen-q4-dspark-p0,code,10,60,0.0000,0.0000,0.0000,0.0000
|
| 22 |
+
qwen-q4-dspark-p0,chat,1,58,0.7089,0.7275,0.6651,0.7472
|
| 23 |
+
qwen-q4-dspark-p0,chat,2,58,0.4922,0.5095,0.4431,0.5365
|
| 24 |
+
qwen-q4-dspark-p0,chat,3,58,0.3525,0.3375,0.3044,0.3973
|
| 25 |
+
qwen-q4-dspark-p0,chat,4,58,0.2589,0.2205,0.2158,0.3017
|
| 26 |
+
qwen-q4-dspark-p0,chat,5,58,0.1988,0.1545,0.1588,0.2404
|
| 27 |
+
qwen-q4-dspark-p0,chat,6,58,0.1552,0.0895,0.1173,0.1929
|
| 28 |
+
qwen-q4-dspark-p0,chat,7,58,0.1229,0.0650,0.0873,0.1596
|
| 29 |
+
qwen-q4-dspark-p0,chat,8,58,0.0000,0.0000,0.0000,0.0000
|
| 30 |
+
qwen-q4-dspark-p0,chat,9,58,0.0000,0.0000,0.0000,0.0000
|
| 31 |
+
qwen-q4-dspark-p0,chat,10,58,0.0000,0.0000,0.0000,0.0000
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p0,math,1,60,0.8351,0.8435,0.8159,0.8546
|
| 3 |
+
qwen-q4-dspark-p0,math,2,60,0.6908,0.6990,0.6574,0.7251
|
| 4 |
+
qwen-q4-dspark-p0,math,3,60,0.5831,0.5715,0.5408,0.6269
|
| 5 |
+
qwen-q4-dspark-p0,math,4,60,0.4884,0.4880,0.4413,0.5375
|
| 6 |
+
qwen-q4-dspark-p0,math,5,60,0.4097,0.3970,0.3609,0.4619
|
| 7 |
+
qwen-q4-dspark-p0,code,1,60,0.7904,0.7920,0.7718,0.8086
|
| 8 |
+
qwen-q4-dspark-p0,code,2,60,0.6033,0.5945,0.5752,0.6310
|
| 9 |
+
qwen-q4-dspark-p0,code,3,60,0.4645,0.4540,0.4331,0.4965
|
| 10 |
+
qwen-q4-dspark-p0,code,4,60,0.3539,0.3290,0.3204,0.3897
|
| 11 |
+
qwen-q4-dspark-p0,code,5,60,0.2637,0.2300,0.2305,0.2998
|
| 12 |
+
qwen-q4-dspark-p0,chat,1,58,0.7086,0.7555,0.6648,0.7460
|
| 13 |
+
qwen-q4-dspark-p0,chat,2,58,0.4859,0.5130,0.4345,0.5313
|
| 14 |
+
qwen-q4-dspark-p0,chat,3,58,0.3529,0.3545,0.3042,0.3996
|
| 15 |
+
qwen-q4-dspark-p0,chat,4,58,0.2674,0.2155,0.2195,0.3135
|
| 16 |
+
qwen-q4-dspark-p0,chat,5,58,0.2048,0.1445,0.1607,0.2485
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-dspark-p0,math,1,60,0.8275,0.8235,0.8088,0.8465
|
| 3 |
+
qwen-q4-dspark-p0,math,2,60,0.6883,0.7065,0.6567,0.7224
|
| 4 |
+
qwen-q4-dspark-p0,math,3,60,0.5771,0.5700,0.5378,0.6177
|
| 5 |
+
qwen-q4-dspark-p0,math,4,60,0.4843,0.4585,0.4372,0.5337
|
| 6 |
+
qwen-q4-dspark-p0,math,5,60,0.4117,0.4000,0.3624,0.4656
|
| 7 |
+
qwen-q4-dspark-p0,math,6,60,0.3543,0.3390,0.3045,0.4083
|
| 8 |
+
qwen-q4-dspark-p0,math,7,60,0.3056,0.2950,0.2568,0.3597
|
| 9 |
+
qwen-q4-dspark-p0,code,1,60,0.7944,0.8080,0.7753,0.8127
|
| 10 |
+
qwen-q4-dspark-p0,code,2,60,0.6135,0.6125,0.5853,0.6421
|
| 11 |
+
qwen-q4-dspark-p0,code,3,60,0.4618,0.4355,0.4318,0.4931
|
| 12 |
+
qwen-q4-dspark-p0,code,4,60,0.3523,0.3255,0.3217,0.3855
|
| 13 |
+
qwen-q4-dspark-p0,code,5,60,0.2684,0.2355,0.2363,0.3034
|
| 14 |
+
qwen-q4-dspark-p0,code,6,60,0.1977,0.1725,0.1691,0.2283
|
| 15 |
+
qwen-q4-dspark-p0,code,7,60,0.1526,0.1190,0.1241,0.1840
|
| 16 |
+
qwen-q4-dspark-p0,chat,1,58,0.7089,0.7275,0.6651,0.7472
|
| 17 |
+
qwen-q4-dspark-p0,chat,2,58,0.4922,0.5095,0.4431,0.5365
|
| 18 |
+
qwen-q4-dspark-p0,chat,3,58,0.3525,0.3375,0.3044,0.3973
|
| 19 |
+
qwen-q4-dspark-p0,chat,4,58,0.2589,0.2205,0.2158,0.3017
|
| 20 |
+
qwen-q4-dspark-p0,chat,5,58,0.1988,0.1545,0.1588,0.2404
|
| 21 |
+
qwen-q4-dspark-p0,chat,6,58,0.1552,0.0895,0.1173,0.1929
|
| 22 |
+
qwen-q4-dspark-p0,chat,7,58,0.1229,0.0650,0.0873,0.1596
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-eagle3,math,1,60,0.6393,0.6360,0.6221,0.6572
|
| 3 |
+
qwen-q4-eagle3,math,2,60,0.4121,0.4130,0.3912,0.4337
|
| 4 |
+
qwen-q4-eagle3,math,3,60,0.2157,0.2145,0.2011,0.2309
|
| 5 |
+
qwen-q4-eagle3,math,4,60,0.1215,0.1165,0.1095,0.1351
|
| 6 |
+
qwen-q4-eagle3,math,5,60,0.0731,0.0705,0.0631,0.0844
|
| 7 |
+
qwen-q4-eagle3,math,6,60,0.0449,0.0380,0.0372,0.0539
|
| 8 |
+
qwen-q4-eagle3,math,7,60,0.0274,0.0225,0.0218,0.0335
|
| 9 |
+
qwen-q4-eagle3,math,8,60,0.0165,0.0110,0.0126,0.0207
|
| 10 |
+
qwen-q4-eagle3,math,9,60,0.0087,0.0085,0.0054,0.0124
|
| 11 |
+
qwen-q4-eagle3,math,10,60,0.0058,0.0000,0.0032,0.0086
|
| 12 |
+
qwen-q4-eagle3,code,1,60,0.6602,0.6415,0.6420,0.6794
|
| 13 |
+
qwen-q4-eagle3,code,2,60,0.4151,0.3990,0.3912,0.4416
|
| 14 |
+
qwen-q4-eagle3,code,3,60,0.2156,0.1740,0.1876,0.2477
|
| 15 |
+
qwen-q4-eagle3,code,4,60,0.1279,0.0925,0.1029,0.1570
|
| 16 |
+
qwen-q4-eagle3,code,5,60,0.0874,0.0460,0.0640,0.1149
|
| 17 |
+
qwen-q4-eagle3,code,6,60,0.0572,0.0200,0.0382,0.0792
|
| 18 |
+
qwen-q4-eagle3,code,7,60,0.0389,0.0100,0.0229,0.0571
|
| 19 |
+
qwen-q4-eagle3,code,8,60,0.0305,0.0090,0.0162,0.0474
|
| 20 |
+
qwen-q4-eagle3,code,9,60,0.0217,0.0000,0.0105,0.0353
|
| 21 |
+
qwen-q4-eagle3,code,10,60,0.0163,0.0000,0.0081,0.0259
|
| 22 |
+
qwen-q4-eagle3,chat,1,58,0.5736,0.6015,0.5236,0.6212
|
| 23 |
+
qwen-q4-eagle3,chat,2,58,0.3468,0.3715,0.2998,0.3917
|
| 24 |
+
qwen-q4-eagle3,chat,3,58,0.1888,0.1630,0.1524,0.2246
|
| 25 |
+
qwen-q4-eagle3,chat,4,58,0.1189,0.0790,0.0884,0.1512
|
| 26 |
+
qwen-q4-eagle3,chat,5,58,0.0831,0.0405,0.0580,0.1095
|
| 27 |
+
qwen-q4-eagle3,chat,6,58,0.0650,0.0285,0.0418,0.0898
|
| 28 |
+
qwen-q4-eagle3,chat,7,58,0.0490,0.0095,0.0291,0.0712
|
| 29 |
+
qwen-q4-eagle3,chat,8,58,0.0407,0.0080,0.0223,0.0618
|
| 30 |
+
qwen-q4-eagle3,chat,9,58,0.0344,0.0000,0.0168,0.0545
|
| 31 |
+
qwen-q4-eagle3,chat,10,58,0.0292,0.0000,0.0136,0.0473
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-eagle3,math,1,60,0.6475,0.6315,0.6282,0.6682
|
| 3 |
+
qwen-q4-eagle3,math,2,60,0.4171,0.4050,0.3944,0.4411
|
| 4 |
+
qwen-q4-eagle3,math,3,60,0.2238,0.2180,0.2056,0.2438
|
| 5 |
+
qwen-q4-eagle3,math,4,60,0.1287,0.1205,0.1123,0.1467
|
| 6 |
+
qwen-q4-eagle3,math,5,60,0.0775,0.0695,0.0650,0.0921
|
| 7 |
+
qwen-q4-eagle3,code,1,60,0.6593,0.6480,0.6400,0.6802
|
| 8 |
+
qwen-q4-eagle3,code,2,60,0.4212,0.4105,0.3942,0.4513
|
| 9 |
+
qwen-q4-eagle3,code,3,60,0.2141,0.1760,0.1854,0.2466
|
| 10 |
+
qwen-q4-eagle3,code,4,60,0.1281,0.0880,0.1010,0.1594
|
| 11 |
+
qwen-q4-eagle3,code,5,60,0.0828,0.0480,0.0581,0.1121
|
| 12 |
+
qwen-q4-eagle3,chat,1,58,0.5732,0.5970,0.5165,0.6252
|
| 13 |
+
qwen-q4-eagle3,chat,2,58,0.3544,0.3435,0.3023,0.4045
|
| 14 |
+
qwen-q4-eagle3,chat,3,58,0.1993,0.1605,0.1593,0.2427
|
| 15 |
+
qwen-q4-eagle3,chat,4,58,0.1258,0.0920,0.0925,0.1614
|
| 16 |
+
qwen-q4-eagle3,chat,5,58,0.0821,0.0435,0.0545,0.1116
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-eagle3,math,1,60,0.6443,0.6260,0.6251,0.6649
|
| 3 |
+
qwen-q4-eagle3,math,2,60,0.4131,0.4030,0.3907,0.4366
|
| 4 |
+
qwen-q4-eagle3,math,3,60,0.2196,0.2180,0.2024,0.2386
|
| 5 |
+
qwen-q4-eagle3,math,4,60,0.1262,0.1180,0.1105,0.1435
|
| 6 |
+
qwen-q4-eagle3,math,5,60,0.0755,0.0675,0.0633,0.0897
|
| 7 |
+
qwen-q4-eagle3,math,6,60,0.0456,0.0360,0.0356,0.0571
|
| 8 |
+
qwen-q4-eagle3,math,7,60,0.0280,0.0200,0.0204,0.0374
|
| 9 |
+
qwen-q4-eagle3,code,1,60,0.6557,0.6445,0.6375,0.6748
|
| 10 |
+
qwen-q4-eagle3,code,2,60,0.4176,0.4125,0.3919,0.4460
|
| 11 |
+
qwen-q4-eagle3,code,3,60,0.2125,0.1815,0.1843,0.2452
|
| 12 |
+
qwen-q4-eagle3,code,4,60,0.1263,0.0875,0.0983,0.1595
|
| 13 |
+
qwen-q4-eagle3,code,5,60,0.0813,0.0490,0.0565,0.1118
|
| 14 |
+
qwen-q4-eagle3,code,6,60,0.0526,0.0190,0.0320,0.0773
|
| 15 |
+
qwen-q4-eagle3,code,7,60,0.0377,0.0100,0.0201,0.0593
|
| 16 |
+
qwen-q4-eagle3,chat,1,58,0.5708,0.6045,0.5146,0.6221
|
| 17 |
+
qwen-q4-eagle3,chat,2,58,0.3497,0.3395,0.2993,0.3989
|
| 18 |
+
qwen-q4-eagle3,chat,3,58,0.1902,0.1580,0.1547,0.2273
|
| 19 |
+
qwen-q4-eagle3,chat,4,58,0.1203,0.0840,0.0906,0.1531
|
| 20 |
+
qwen-q4-eagle3,chat,5,58,0.0753,0.0425,0.0509,0.1010
|
| 21 |
+
qwen-q4-eagle3,chat,6,58,0.0579,0.0205,0.0352,0.0826
|
| 22 |
+
qwen-q4-eagle3,chat,7,58,0.0427,0.0090,0.0227,0.0639
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-vanilla17b,math,1,60,0.8606,0.8640,0.8454,0.8754
|
| 3 |
+
qwen-q4-vanilla17b,math,2,60,0.7416,0.7255,0.7152,0.7684
|
| 4 |
+
qwen-q4-vanilla17b,math,3,60,0.6567,0.6505,0.6250,0.6896
|
| 5 |
+
qwen-q4-vanilla17b,math,4,60,0.5859,0.5725,0.5521,0.6207
|
| 6 |
+
qwen-q4-vanilla17b,math,5,60,0.5293,0.5120,0.4932,0.5667
|
| 7 |
+
qwen-q4-vanilla17b,math,6,60,0.4771,0.4690,0.4406,0.5155
|
| 8 |
+
qwen-q4-vanilla17b,math,7,60,0.4277,0.4200,0.3902,0.4668
|
| 9 |
+
qwen-q4-vanilla17b,math,8,60,0.3844,0.3660,0.3468,0.4236
|
| 10 |
+
qwen-q4-vanilla17b,math,9,60,0.3481,0.3410,0.3101,0.3869
|
| 11 |
+
qwen-q4-vanilla17b,math,10,60,0.3208,0.3140,0.2823,0.3584
|
| 12 |
+
qwen-q4-vanilla17b,code,1,60,0.8284,0.8350,0.8096,0.8477
|
| 13 |
+
qwen-q4-vanilla17b,code,2,60,0.6865,0.6875,0.6631,0.7102
|
| 14 |
+
qwen-q4-vanilla17b,code,3,60,0.5856,0.5780,0.5595,0.6142
|
| 15 |
+
qwen-q4-vanilla17b,code,4,60,0.5046,0.4840,0.4784,0.5369
|
| 16 |
+
qwen-q4-vanilla17b,code,5,60,0.4359,0.4150,0.4082,0.4698
|
| 17 |
+
qwen-q4-vanilla17b,code,6,60,0.3831,0.3645,0.3531,0.4188
|
| 18 |
+
qwen-q4-vanilla17b,code,7,60,0.3410,0.3225,0.3108,0.3766
|
| 19 |
+
qwen-q4-vanilla17b,code,8,60,0.3017,0.2780,0.2722,0.3373
|
| 20 |
+
qwen-q4-vanilla17b,code,9,60,0.2707,0.2420,0.2401,0.3074
|
| 21 |
+
qwen-q4-vanilla17b,code,10,60,0.2394,0.2145,0.2116,0.2720
|
| 22 |
+
qwen-q4-vanilla17b,chat,1,58,0.7707,0.7445,0.7471,0.7945
|
| 23 |
+
qwen-q4-vanilla17b,chat,2,58,0.5909,0.5880,0.5608,0.6219
|
| 24 |
+
qwen-q4-vanilla17b,chat,3,58,0.4747,0.4780,0.4397,0.5127
|
| 25 |
+
qwen-q4-vanilla17b,chat,4,58,0.3836,0.3620,0.3472,0.4236
|
| 26 |
+
qwen-q4-vanilla17b,chat,5,58,0.3246,0.2845,0.2864,0.3674
|
| 27 |
+
qwen-q4-vanilla17b,chat,6,58,0.2791,0.2310,0.2390,0.3239
|
| 28 |
+
qwen-q4-vanilla17b,chat,7,58,0.2419,0.1910,0.2015,0.2864
|
| 29 |
+
qwen-q4-vanilla17b,chat,8,58,0.2197,0.1640,0.1788,0.2661
|
| 30 |
+
qwen-q4-vanilla17b,chat,9,58,0.1945,0.1380,0.1544,0.2391
|
| 31 |
+
qwen-q4-vanilla17b,chat,10,58,0.1752,0.1145,0.1350,0.2199
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-vanilla17b,math,1,60,0.8614,0.8695,0.8469,0.8759
|
| 3 |
+
qwen-q4-vanilla17b,math,2,60,0.7467,0.7370,0.7221,0.7711
|
| 4 |
+
qwen-q4-vanilla17b,math,3,60,0.6642,0.6555,0.6354,0.6929
|
| 5 |
+
qwen-q4-vanilla17b,math,4,60,0.5917,0.5945,0.5596,0.6233
|
| 6 |
+
qwen-q4-vanilla17b,math,5,60,0.5365,0.5255,0.5022,0.5705
|
| 7 |
+
qwen-q4-vanilla17b,code,1,60,0.8306,0.8225,0.8161,0.8458
|
| 8 |
+
qwen-q4-vanilla17b,code,2,60,0.6994,0.6960,0.6791,0.7209
|
| 9 |
+
qwen-q4-vanilla17b,code,3,60,0.5953,0.5850,0.5719,0.6201
|
| 10 |
+
qwen-q4-vanilla17b,code,4,60,0.5117,0.4920,0.4882,0.5382
|
| 11 |
+
qwen-q4-vanilla17b,code,5,60,0.4440,0.4175,0.4158,0.4744
|
| 12 |
+
qwen-q4-vanilla17b,chat,1,58,0.7753,0.7690,0.7534,0.7985
|
| 13 |
+
qwen-q4-vanilla17b,chat,2,58,0.6074,0.5805,0.5755,0.6407
|
| 14 |
+
qwen-q4-vanilla17b,chat,3,58,0.4889,0.4695,0.4515,0.5299
|
| 15 |
+
qwen-q4-vanilla17b,chat,4,58,0.4014,0.3630,0.3599,0.4463
|
| 16 |
+
qwen-q4-vanilla17b,chat,5,58,0.3373,0.2830,0.2937,0.3847
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q4-vanilla17b,math,1,60,0.8568,0.8570,0.8415,0.8715
|
| 3 |
+
qwen-q4-vanilla17b,math,2,60,0.7372,0.7320,0.7132,0.7605
|
| 4 |
+
qwen-q4-vanilla17b,math,3,60,0.6551,0.6560,0.6267,0.6840
|
| 5 |
+
qwen-q4-vanilla17b,math,4,60,0.5800,0.5900,0.5486,0.6121
|
| 6 |
+
qwen-q4-vanilla17b,math,5,60,0.5246,0.5210,0.4935,0.5572
|
| 7 |
+
qwen-q4-vanilla17b,math,6,60,0.4753,0.4745,0.4427,0.5094
|
| 8 |
+
qwen-q4-vanilla17b,math,7,60,0.4283,0.4170,0.3954,0.4638
|
| 9 |
+
qwen-q4-vanilla17b,code,1,60,0.8222,0.8260,0.8074,0.8375
|
| 10 |
+
qwen-q4-vanilla17b,code,2,60,0.6858,0.6730,0.6650,0.7075
|
| 11 |
+
qwen-q4-vanilla17b,code,3,60,0.5783,0.5640,0.5553,0.6035
|
| 12 |
+
qwen-q4-vanilla17b,code,4,60,0.4988,0.4785,0.4743,0.5257
|
| 13 |
+
qwen-q4-vanilla17b,code,5,60,0.4323,0.4150,0.4042,0.4638
|
| 14 |
+
qwen-q4-vanilla17b,code,6,60,0.3756,0.3580,0.3435,0.4094
|
| 15 |
+
qwen-q4-vanilla17b,code,7,60,0.3322,0.3050,0.3009,0.3676
|
| 16 |
+
qwen-q4-vanilla17b,chat,1,58,0.7715,0.7615,0.7502,0.7939
|
| 17 |
+
qwen-q4-vanilla17b,chat,2,58,0.5965,0.5805,0.5665,0.6277
|
| 18 |
+
qwen-q4-vanilla17b,chat,3,58,0.4779,0.4650,0.4422,0.5168
|
| 19 |
+
qwen-q4-vanilla17b,chat,4,58,0.3930,0.3585,0.3511,0.4369
|
| 20 |
+
qwen-q4-vanilla17b,chat,5,58,0.3258,0.2680,0.2832,0.3719
|
| 21 |
+
qwen-q4-vanilla17b,chat,6,58,0.2849,0.2140,0.2400,0.3335
|
| 22 |
+
qwen-q4-vanilla17b,chat,7,58,0.2500,0.1795,0.2042,0.3002
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q8-eagle3,math,1,60,0.6384,0.6310,0.6204,0.6561
|
| 3 |
+
qwen-q8-eagle3,math,2,60,0.4114,0.4060,0.3919,0.4320
|
| 4 |
+
qwen-q8-eagle3,math,3,60,0.2123,0.2140,0.1969,0.2300
|
| 5 |
+
qwen-q8-eagle3,math,4,60,0.1168,0.1140,0.1051,0.1296
|
| 6 |
+
qwen-q8-eagle3,math,5,60,0.0649,0.0620,0.0564,0.0746
|
| 7 |
+
qwen-q8-eagle3,math,6,60,0.0355,0.0315,0.0295,0.0418
|
| 8 |
+
qwen-q8-eagle3,math,7,60,0.0199,0.0190,0.0161,0.0240
|
| 9 |
+
qwen-q8-eagle3,math,8,60,0.0110,0.0100,0.0084,0.0138
|
| 10 |
+
qwen-q8-eagle3,math,9,60,0.0068,0.0040,0.0049,0.0088
|
| 11 |
+
qwen-q8-eagle3,math,10,60,0.0043,0.0000,0.0027,0.0059
|
| 12 |
+
qwen-q8-eagle3,code,1,60,0.6466,0.6500,0.6291,0.6649
|
| 13 |
+
qwen-q8-eagle3,code,2,60,0.4087,0.4010,0.3883,0.4312
|
| 14 |
+
qwen-q8-eagle3,code,3,60,0.1994,0.1765,0.1784,0.2247
|
| 15 |
+
qwen-q8-eagle3,code,4,60,0.1104,0.0920,0.0901,0.1367
|
| 16 |
+
qwen-q8-eagle3,code,5,60,0.0654,0.0450,0.0475,0.0899
|
| 17 |
+
qwen-q8-eagle3,code,6,60,0.0408,0.0200,0.0257,0.0614
|
| 18 |
+
qwen-q8-eagle3,code,7,60,0.0283,0.0100,0.0170,0.0426
|
| 19 |
+
qwen-q8-eagle3,code,8,60,0.0205,0.0080,0.0114,0.0317
|
| 20 |
+
qwen-q8-eagle3,code,9,60,0.0127,0.0000,0.0064,0.0214
|
| 21 |
+
qwen-q8-eagle3,code,10,60,0.0092,0.0000,0.0035,0.0173
|
| 22 |
+
qwen-q8-eagle3,chat,1,58,0.5779,0.6090,0.5295,0.6233
|
| 23 |
+
qwen-q8-eagle3,chat,2,58,0.3406,0.3450,0.2972,0.3811
|
| 24 |
+
qwen-q8-eagle3,chat,3,58,0.1728,0.1625,0.1398,0.2089
|
| 25 |
+
qwen-q8-eagle3,chat,4,58,0.1127,0.0845,0.0830,0.1480
|
| 26 |
+
qwen-q8-eagle3,chat,5,58,0.0690,0.0455,0.0468,0.0959
|
| 27 |
+
qwen-q8-eagle3,chat,6,58,0.0538,0.0280,0.0343,0.0776
|
| 28 |
+
qwen-q8-eagle3,chat,7,58,0.0381,0.0095,0.0232,0.0562
|
| 29 |
+
qwen-q8-eagle3,chat,8,58,0.0309,0.0090,0.0170,0.0477
|
| 30 |
+
qwen-q8-eagle3,chat,9,58,0.0237,0.0000,0.0121,0.0382
|
| 31 |
+
qwen-q8-eagle3,chat,10,58,0.0181,0.0000,0.0079,0.0312
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q8-eagle3,math,1,60,0.6440,0.6290,0.6264,0.6617
|
| 3 |
+
qwen-q8-eagle3,math,2,60,0.4159,0.4220,0.3966,0.4358
|
| 4 |
+
qwen-q8-eagle3,math,3,60,0.2202,0.2165,0.2050,0.2356
|
| 5 |
+
qwen-q8-eagle3,math,4,60,0.1221,0.1155,0.1107,0.1348
|
| 6 |
+
qwen-q8-eagle3,math,5,60,0.0690,0.0630,0.0602,0.0790
|
| 7 |
+
qwen-q8-eagle3,code,1,60,0.6543,0.6465,0.6373,0.6724
|
| 8 |
+
qwen-q8-eagle3,code,2,60,0.4166,0.4120,0.3953,0.4391
|
| 9 |
+
qwen-q8-eagle3,code,3,60,0.2039,0.1845,0.1819,0.2286
|
| 10 |
+
qwen-q8-eagle3,code,4,60,0.1145,0.0900,0.0935,0.1394
|
| 11 |
+
qwen-q8-eagle3,code,5,60,0.0663,0.0450,0.0489,0.0882
|
| 12 |
+
qwen-q8-eagle3,chat,1,58,0.5884,0.6265,0.5389,0.6331
|
| 13 |
+
qwen-q8-eagle3,chat,2,58,0.3518,0.3700,0.3078,0.3941
|
| 14 |
+
qwen-q8-eagle3,chat,3,58,0.1824,0.1740,0.1497,0.2200
|
| 15 |
+
qwen-q8-eagle3,chat,4,58,0.1178,0.0885,0.0903,0.1517
|
| 16 |
+
qwen-q8-eagle3,chat,5,58,0.0723,0.0445,0.0506,0.0976
|
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
|
| 2 |
+
qwen-q8-eagle3,math,1,60,0.6422,0.6315,0.6244,0.6602
|
| 3 |
+
qwen-q8-eagle3,math,2,60,0.4124,0.4095,0.3926,0.4324
|
| 4 |
+
qwen-q8-eagle3,math,3,60,0.2172,0.2140,0.2027,0.2328
|
| 5 |
+
qwen-q8-eagle3,math,4,60,0.1193,0.1155,0.1080,0.1315
|
| 6 |
+
qwen-q8-eagle3,math,5,60,0.0672,0.0630,0.0587,0.0769
|
| 7 |
+
qwen-q8-eagle3,math,6,60,0.0396,0.0355,0.0337,0.0458
|
| 8 |
+
qwen-q8-eagle3,math,7,60,0.0225,0.0195,0.0182,0.0270
|
| 9 |
+
qwen-q8-eagle3,code,1,60,0.6502,0.6410,0.6335,0.6674
|
| 10 |
+
qwen-q8-eagle3,code,2,60,0.4082,0.4040,0.3865,0.4310
|
| 11 |
+
qwen-q8-eagle3,code,3,60,0.1971,0.1840,0.1773,0.2205
|
| 12 |
+
qwen-q8-eagle3,code,4,60,0.1089,0.0880,0.0905,0.1322
|
| 13 |
+
qwen-q8-eagle3,code,5,60,0.0638,0.0430,0.0474,0.0862
|
| 14 |
+
qwen-q8-eagle3,code,6,60,0.0379,0.0200,0.0249,0.0543
|
| 15 |
+
qwen-q8-eagle3,code,7,60,0.0276,0.0090,0.0166,0.0418
|
| 16 |
+
qwen-q8-eagle3,chat,1,58,0.5867,0.6200,0.5382,0.6317
|
| 17 |
+
qwen-q8-eagle3,chat,2,58,0.3437,0.3615,0.3013,0.3834
|
| 18 |
+
qwen-q8-eagle3,chat,3,58,0.1738,0.1685,0.1435,0.2065
|
| 19 |
+
qwen-q8-eagle3,chat,4,58,0.1146,0.0855,0.0876,0.1466
|
| 20 |
+
qwen-q8-eagle3,chat,5,58,0.0699,0.0415,0.0478,0.0961
|
| 21 |
+
qwen-q8-eagle3,chat,6,58,0.0515,0.0190,0.0318,0.0751
|
| 22 |
+
qwen-q8-eagle3,chat,7,58,0.0363,0.0095,0.0214,0.0545
|