reyden009 commited on
Commit
d4c2896
·
verified ·
1 Parent(s): 384ebd3

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +6 -0
  2. .gitignore +40 -0
  3. .python-version +1 -0
  4. AGENTS.md +71 -0
  5. LICENSE +21 -0
  6. Makefile +32 -0
  7. README.md +71 -0
  8. experiments/analysis/README.md +66 -0
  9. experiments/analysis/checks.md +33 -0
  10. experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv +10 -0
  11. experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv +10 -0
  12. experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv +10 -0
  13. experiments/analysis/curves/acc_by_pos_curves-gemma-q4-mtp.csv +10 -0
  14. experiments/analysis/curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv +10 -0
  15. experiments/analysis/curves/acc_by_pos_curves-gemma-q5-mtp.csv +10 -0
  16. experiments/analysis/curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv +10 -0
  17. experiments/analysis/curves/acc_by_pos_curves-gemma-q8-mtp.csv +10 -0
  18. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dflash.csv +10 -0
  19. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv +10 -0
  20. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv +10 -0
  21. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv +10 -0
  22. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv +10 -0
  23. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-eagle3.csv +10 -0
  24. experiments/analysis/curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv +10 -0
  25. experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dflash.csv +10 -0
  26. experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv +10 -0
  27. experiments/analysis/curves/acc_by_pos_curves-qwen-q5-eagle3.csv +10 -0
  28. experiments/analysis/curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv +10 -0
  29. experiments/analysis/curves/acc_by_pos_curves-qwen-q8-eagle3.csv +10 -0
  30. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv +31 -0
  31. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv +16 -0
  32. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv +22 -0
  33. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv +31 -0
  34. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv +16 -0
  35. experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv +22 -0
  36. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv +31 -0
  37. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv +16 -0
  38. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv +22 -0
  39. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv +31 -0
  40. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv +16 -0
  41. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv +22 -0
  42. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv +31 -0
  43. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv +16 -0
  44. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv +22 -0
  45. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv +31 -0
  46. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv +16 -0
  47. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv +22 -0
  48. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv +31 -0
  49. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv +16 -0
  50. experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv +22 -0
.gitattributes CHANGED
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ paper/figures/F1a_acceptance_qwen.png filter=lfs diff=lfs merge=lfs -text
37
+ paper/figures/F1b_acceptance_gemma.png filter=lfs diff=lfs merge=lfs -text
38
+ paper/figures/F2_speedup.png filter=lfs diff=lfs merge=lfs -text
39
+ paper/figures/F3_breakeven.png filter=lfs diff=lfs merge=lfs -text
40
+ paper/figures/F4_alpha_position_ksweep.png filter=lfs diff=lfs merge=lfs -text
41
+ paper/paper.pdf filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # modelos grandes y artefactos pesados (se bajan por script, no se commitean)
2
+ models/
3
+ *.gguf
4
+ *.safetensors
5
+ *.bin
6
+
7
+ # papers y notas del corpus: se regeneran con scripts/download_papers.py, no se versionan
8
+ papers/
9
+ notes/
10
+ scripts/download_papers.py
11
+
12
+ # datos de experimentos crudos (se regeneran; los resultados limpios sí se commitean)
13
+ experiments/runs/
14
+ experiments/prompts/raw/
15
+
16
+ # python (uv)
17
+ .venv/
18
+ __pycache__/
19
+ *.pyc
20
+
21
+ # entorno local (no parte del proyecto)
22
+ .atl/
23
+
24
+ # el paper (manuscript/) y la documentación interna de investigación (docs/) no se versionan:
25
+ # el paper se publica por separado (arXiv/HF) y el repo público es su companion
26
+ manuscript/
27
+ docs/
28
+
29
+ # estado vivo del proyecto (bitácora local, no se versiona)
30
+ STATUS.md
31
+
32
+ # tests locales (no se versionan)
33
+ tests/
34
+
35
+ # editores/OS
36
+ .DS_Store
37
+ *.swp
38
+ tests/test_smoke.py
39
+ tests/conftest.py
40
+ tests/
.python-version ADDED
@@ -0,0 +1 @@
 
 
1
+ 3.12
AGENTS.md ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AGENTS.md — Rules for agents (and humans) working in this repo
2
+
3
+ This repository is a **research project** with a master plan and reproducible
4
+ protocol. Before touching code or experiments, read this completely.
5
+
6
+ ## 1. Context in one sentence
7
+
8
+ Empirical study of modern speculative decoding (EAGLE-3/DFlash/DSpark) on an
9
+ RTX 4060 Ti 16 GB with Gemma 4 12B and Qwen3-8B GGUF: acceptance by domain,
10
+ quantization and hardware break-even. The research documentation (thesis, gaps,
11
+ plan) lives locally in `docs/` (not versioned); live status is ALWAYS in
12
+ `STATUS.md` (local, not versioned).
13
+
14
+ ## 2. Conventions
15
+
16
+ - **Language**: docs, logs and commit messages in **English**; code and
17
+ technical metadata in **English**. Do not mix languages within one file.
18
+ - **Documentation**: markdown and session logs in `docs/`, live status in
19
+ `STATUS.md` — all local, **not versioned**. The corpus `papers/` and its
20
+ notes `notes/` are not versioned (regenerated locally with a download script,
21
+ outside the repo).
22
+ - **Environment**: activate with `source scripts/spec-env.sh` (uv + CUDA in
23
+ `~/cuda`). Never install anything with sudo; if privileges are needed, ask
24
+ uli and wait.
25
+ - **Minimum quality before commit**: `ruff check`, `ruff format --check`,
26
+ `pyright` with no errors on what you touched.
27
+
28
+ ## 3. Git rules
29
+
30
+ - **Logical commits** (never a mega-commit); concise messages in English with a
31
+ type prefix (`docs:`, `chore:`, `feat:`, `fix:`).
32
+ - **Do not version**: `models/`, `*.gguf`, `papers/`, `notes/`,
33
+ `experiments/runs/`, `.venv/`, `.atl/` (see `.gitignore`).
34
+ - Do not commit regenerable artifacts (PDFs, raw runs, build logs).
35
+ - Before committing: `git status` + `git diff` to review what goes in.
36
+
37
+ ## 4. Experiments (Phase 1+)
38
+
39
+ - **Never** launch a run without a viable `--resume` (incremental JSONL) unless
40
+ explicitly decided.
41
+ - Follow the master plan protocol (§4): fixed seeds (42), documented temperature,
42
+ target-only baseline with the SAME quantization, config.json + metrics.json per
43
+ run in `experiments/runs/<id>/`.
44
+ - **Do not** use the full DeepSpec data cache (~38 TB); only subsamples.
45
+ - Do not change the master plan targets/decisions (§10) without uli's approval.
46
+
47
+ ## 5. Sessions and handoff (mandatory)
48
+
49
+ At the end of any work session:
50
+
51
+ 1. Write the session log at `docs/logs/YYYY-MM-DD-sesion-NN-asunto.md` (local
52
+ logbook, not versioned: what was done, decisions, pending items, what is
53
+ still running).
54
+ 2. Update `STATUS.md` (local, not versioned: current phase, what is running,
55
+ pending, next steps).
56
+ 3. Save in engram (`mem_save`/`mem_session_summary`) non-obvious decisions and
57
+ discoveries (project: `spectulative_decoding`).
58
+ 4. If background processes remain (downloads, builds, runs): note them in
59
+ STATUS.md (local) with PID and log.
60
+
61
+ A new agent opening the repo should read: `AGENTS.md` → (local `STATUS.md`,
62
+ docs and logs) before acting.
63
+
64
+ ## 6. Forbidden / careful
65
+
66
+ - **Forbidden**: sudo without notice; versioning models/PDFs/runs; using the
67
+ 38 TB cache; modifying raw results; deleting another session's
68
+ `experiments/runs/`.
69
+ - **Careful**: recompiling llama.cpp without noting the exact commit; changing
70
+ the overnight/resume regime; discarding speedup < 1 results (they are valid
71
+ data, not errors).
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Speculative Decoding Lab contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
Makefile ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Makefile — common lab tasks
2
+ # Requires uv in PATH (~/.local/bin/uv).
3
+
4
+ .PHONY: setup lint fmt typecheck models status clean
5
+
6
+ setup: ## Full environment (venv + CUDA toolkit)
7
+ uv sync
8
+ bash scripts/prepare-cuda.sh || true
9
+
10
+ lint: ## ruff check
11
+ uv run ruff check scripts/
12
+
13
+ fmt: ## ruff format (applies)
14
+ uv run ruff format scripts/
15
+
16
+ fmt-check: ## ruff format --check
17
+ uv run ruff format --check scripts/
18
+
19
+ typecheck: ## pyright
20
+ uv run pyright scripts/
21
+
22
+ models: ## List available GGUFs in models/
23
+ ls -lh models/
24
+
25
+ status: ## Live project status
26
+ @cat STATUS.md
27
+
28
+ clean: ## Remove regenerable artifacts (does NOT delete models/)
29
+ rm -rf experiments/runs/*
30
+
31
+ help:
32
+ @grep -E '^[a-zA-Z-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}'
README.md ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Speculative Decoding Lab
2
+
3
+ Systematic, reproducible empirical evaluation of **modern speculative decoding**
4
+ (EAGLE-3, DFlash, DSpark vs. vanilla and model-free) on an **RTX 4060 Ti 16 GB**
5
+ with **Gemma 4 12B** and **Qwen3-8B** GGUF-quantized models: acceptance by
6
+ cognitive domain, quantization effects, and hardware break-even maps.
7
+
8
+ > Paper: [PDF](paper/paper.pdf) · [arXiv-ready source bundle](paper/arxiv-bundle.tar.gz) · [source (markdown)](paper/paper.md)
9
+
10
+ ## Hardware used
11
+
12
+ The study was developed and benchmarked on this machine, so all reported
13
+ numbers (speedups, acceptance, break-even) are tied to these specs. The code
14
+ itself is portable: it only needs llama.cpp (CUDA build) and enough VRAM for
15
+ the target model, and can be re-run on any other hardware.
16
+
17
+ | Component | Detail |
18
+ | --- | --- |
19
+ | GPU | NVIDIA RTX 4060 Ti **16 GB** (Ada Lovelace, compute 8.9) |
20
+ | CPU | Intel i5-12400 (6C/12T, 4.4 GHz) |
21
+ | RAM / Disk | 31 GB / ~185 GB free |
22
+ | OS | Ubuntu 24.04 · driver 595.84 |
23
+
24
+ Environment principle: **everything user-space, no sudo** (uv + CUDA runfile in `$HOME`).
25
+
26
+ ## Stack
27
+
28
+ - **Python 3.12 + uv** — `pyproject.toml` + `uv.lock` (numpy, ruff, pyright, cmake, ninja).
29
+ - **CUDA toolkit 12.8.1** in `~/cuda` (runfile `--toolkitpath`). *Note: the `nvcc` binary is not on PyPI (the wheels only ship ptxas).*
30
+ - **llama.cpp master 0713275 (v22)** (CUDA build) in `~/llama.cpp` (built from source; no CUDA binaries are published for Linux).
31
+ - **GGUF models** in `models/` (not versioned; downloaded with `curl -C -`).
32
+
33
+ ## Setup (once)
34
+
35
+ ```bash
36
+ uv sync # create .venv with all dependencies
37
+ source scripts/spec-env.sh # PATH + CUDA_HOME + env vars (idempotent)
38
+ ```
39
+
40
+ ## Quality (required before committing)
41
+
42
+ ```bash
43
+ uv run ruff check . # lint
44
+ uv run ruff format --check . # formatting
45
+ uv run pyright # types
46
+ ```
47
+
48
+ ## Execution regime
49
+
50
+ Overnight sessions with **pause/resume**: `scripts/bench_accept.py --resume`
51
+ (incremental JSONL per prompt; server-based runner with α/τ), downloads with
52
+ `scripts/dl.py`, runs in tmux/nohup. Measurement campaigns are launched with
53
+ `scripts/run_final_chain.sh`, `scripts/run_curves_chain.sh` and
54
+ `scripts/run_ksweep_chain.sh`; final analysis with `scripts/analyze_final.py`.
55
+
56
+ ## Structure
57
+
58
+ ```
59
+ scripts/ environment, downloads, benchmark runner and analysis (bench_accept.py, analyze_final.py)
60
+ experiments/ runs/ (raw, not versioned) and prompts/ (JSONL subsamples)
61
+ models/ GGUFs (not versioned)
62
+ ```
63
+
64
+ ## For agents
65
+
66
+ Read [`AGENTS.md`](AGENTS.md) before working in this repo (conventions, git
67
+ rules, sessions and handoff).
68
+
69
+ ## License
70
+
71
+ MIT — see [`LICENSE`](LICENSE).
experiments/analysis/README.md ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Final analysis - `experiments/analysis/`
2
+
3
+ Generated by `scripts/analyze_final.py` (Python 3.12, stdlib only + numpy + matplotlib).
4
+
5
+ ## Methodology
6
+
7
+ ### Sources
8
+
9
+ - `experiments/runs/<run>/results.jsonl` - one OK record per line (schema in the
10
+ inventory). `config.json` normalizes the run: family (`final/curves/ksweep`),
11
+ target (`{qwen|gemma}-{q4|q5|q8}` from the model path), drafter from
12
+ `spec_type` + `p_min` (dspark) + draft quantization (dflash-f16/q4/q8).
13
+ - `server.log` from `curves-*`/`ksweep-*` runs - `draft acceptance` and
14
+ `acc per pos` lines (only place with per-position curves; the `final-*` runs
15
+ rotated logs and their original segment has no acceptance lines).
16
+
17
+ ### Exclusions (timing sentinels)
18
+
19
+ llama-server reports `tok_per_s = 1,000,000` / `predicted_ms = 0` in 271 records
20
+ from Gemma runs (timing quirk, not real speedup). **Rule applied in every
21
+ calculation**: exclude any record with `tok_per_s >= 1e5` or `predicted_ms <= 0`
22
+ (together with `alpha/tau/draft_n = None`). Qwen has no sentinels. The Gemma
23
+ `metrics.json` files are contaminated by this (mean 12,291 tok/s) and are **not**
24
+ used for tok/s; only `duration_s` and `errors` are used. Counts per run in
25
+ `summary.json` -> `meta.exclusions_per_run` (q4: 18/run, q5: 25/run, q8: 20/run
26
+ in final; 3-4/run in gemma curves/ksweep).
27
+
28
+ ### Log <-> record mapping (curves/ksweep)
29
+
30
+ The order of the `draft acceptance` lines in `server.log` == order of records
31
+ with non-None `alpha` in `results.jsonl` (verified in 41/41 runs, max |diff| =
32
+ 5e-5, rounding only). Gemma sentinel records (alpha=None) have no line and are
33
+ skipped. This assigns the domain to each line. No mismatches in any run.
34
+
35
+ ### Speedup vs solo
36
+
37
+ Per-prompt ratio `tps_draft / tps_solo` matched by `id` against the `solo` run
38
+ of the same family+quant; mean and median of the ratio are reported, plus the
39
+ ratio of means (`agg`). The 6 solo runs (qwen q4/q5/q8, gemma q4/q5/q8) exist
40
+ and are the baseline.
41
+
42
+ ### Break-even alpha_be (paper #32, Bielik et al.)
43
+
44
+ Per `ksweep-*` run and domain (and aggregated): OLS of `TPS = a + beta*alpha`
45
+ over the run records (alpha and per-prompt tok/s, clean records with alpha).
46
+ `alpha_be = (TPS_base - a) / beta` where `TPS_base` = mean tok/s of the
47
+ corresponding `solo` run (same family+quant+domain, final run). CI95 by delta
48
+ method over the OLS covariance of (a, beta). alpha_be > 1 = no reachable
49
+ break-even; < 0 = always above baseline. Paper #32 reports k=2: 38.0-52.8% and
50
+ k=4: 77.7-90.1% (Table 5); the compact 40-77% range covers k=2..4.
51
+
52
+ ## Files
53
+
54
+ - `summary.json` - consolidated numbers (per config x domain and breakeven block).
55
+ - `tables/t2_speedup.md` ... `tables/t6_breakeven.md` - markdown tables for the paper.
56
+ - `curves/acc_by_pos_<run>.csv` - per-position acceptance per domain (41 runs).
57
+ - `checks.md` - result of the checks vs session 08.
58
+ - `manuscript/figures/F1..F4_*.png` - figures (300 dpi; local paper output, `manuscript/` is not versioned).
59
+
60
+ ## Execution
61
+
62
+ ```
63
+ .venv/bin/python scripts/analyze_final.py \
64
+ --runs-dir experiments/runs --out-dir experiments/analysis \
65
+ --figs-dir manuscript/figures
66
+ ```
experiments/analysis/checks.md ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CHECKS - expected values (session 08) vs computed
2
+
3
+ Tolerance: >5% relative deviation -> warning. Speedups are compared against
4
+ the mean and median of the per-prompt ratio; either one passing is enough.
5
+
6
+ | Check | Expected | Computed | Pass? | Note |
7
+ | ------- | ---------- | ----------- | ------- | ------ |
8
+ | qwen-q4 solo | 53.400 | 53.368 | OK | |
9
+ | vanilla17b | 75.500 | 75.453 | OK | |
10
+ | vanilla17b x | 1.410 | 1.41 (med 1.41) | OK | mean 0.3% off, median 0.3% off |
11
+ | vanilla17b alpha | 0.725 | 0.725 | OK | |
12
+ | eagle3 | 74.400 | 74.371 | OK | |
13
+ | eagle3 x | 1.390 | 1.39 (med 1.37) | OK | mean 0.3% off, median 1.2% off |
14
+ | eagle3 alpha | 0.440 | 0.440 | OK | |
15
+ | dspark-p0 | 87.700 | 87.617 | OK | |
16
+ | dspark-p0 x | 1.640 | 1.64 (med 1.63) | OK | mean 0.1% off, median 0.8% off |
17
+ | dspark-p0 alpha | 0.616 | 0.616 | OK | |
18
+ | dspark-p6 | 80.800 | 80.731 | OK | |
19
+ | dspark-p6 x | 1.510 | 1.51 (med 1.50) | OK | mean 0.2% off, median 0.8% off |
20
+ | dspark-p6 alpha | 0.714 | 0.714 | OK | |
21
+ | qwen-q5 solo | 46.800 | 46.800 | OK | |
22
+ | q5 eagle3 | 68.300 | 68.331 | OK | |
23
+ | q5 eagle3 x | 1.460 | 1.46 (med 1.44) | OK | mean 0.0% off, median 1.4% off |
24
+ | q5 dspark-p0 | 80.700 | 80.641 | OK | |
25
+ | q5 dspark-p0 x | 1.720 | 1.72 (med 1.72) | OK | mean 0.2% off, median 0.3% off |
26
+ | qwen-q8 solo | 32.400 | 32.345 | OK | |
27
+ | q8 eagle3 | 52.800 | 52.812 | OK | |
28
+ | q8 eagle3 x | 1.630 | 1.63 (med 1.61) | OK | mean 0.2% off, median 1.0% off |
29
+ | gemma-q4 solo | 34.900 | 34.661 | OK | |
30
+ | gemma-q4 dflash-f16 x | 2.240 | 2.09 (med 2.19) | OK | mean 6.8% off, median 2.4% off |
31
+ | gemma-q4 mtp x | 2.500 | 2.39 (med 2.47) | OK | mean 4.4% off, median 1.0% off |
32
+
33
+ **24/24 checks pass.**
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-f16,math,1,60,0.8875,0.8965,0.8731,0.9004
3
+ gemma-q4-dflash-f16,math,2,60,0.7715,0.7805,0.7496,0.7916
4
+ gemma-q4-dflash-f16,math,3,60,0.6352,0.6340,0.6085,0.6602
5
+ gemma-q4-dflash-f16,code,1,60,0.8467,0.8600,0.8262,0.8678
6
+ gemma-q4-dflash-f16,code,2,60,0.7134,0.7145,0.6828,0.7452
7
+ gemma-q4-dflash-f16,code,3,60,0.5915,0.5775,0.5538,0.6293
8
+ gemma-q4-dflash-f16,chat,1,55,0.6516,0.6730,0.5918,0.7056
9
+ gemma-q4-dflash-f16,chat,2,55,0.4742,0.4350,0.4116,0.5377
10
+ gemma-q4-dflash-f16,chat,3,55,0.3220,0.2410,0.2602,0.3879
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-q4,math,1,60,0.8841,0.8940,0.8703,0.8966
3
+ gemma-q4-dflash-q4,math,2,60,0.7655,0.7855,0.7416,0.7874
4
+ gemma-q4-dflash-q4,math,3,60,0.6332,0.6340,0.6061,0.6592
5
+ gemma-q4-dflash-q4,code,1,60,0.8460,0.8535,0.8244,0.8675
6
+ gemma-q4-dflash-q4,code,2,60,0.7023,0.6975,0.6701,0.7339
7
+ gemma-q4-dflash-q4,code,3,60,0.5776,0.5865,0.5409,0.6155
8
+ gemma-q4-dflash-q4,chat,1,55,0.6411,0.6920,0.5845,0.6937
9
+ gemma-q4-dflash-q4,chat,2,55,0.4609,0.4530,0.4002,0.5217
10
+ gemma-q4-dflash-q4,chat,3,55,0.3073,0.2500,0.2477,0.3702
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-q8,math,1,60,0.8858,0.8965,0.8707,0.8988
3
+ gemma-q4-dflash-q8,math,2,60,0.7701,0.7805,0.7481,0.7899
4
+ gemma-q4-dflash-q8,math,3,60,0.6345,0.6365,0.6085,0.6599
5
+ gemma-q4-dflash-q8,code,1,60,0.8447,0.8590,0.8248,0.8653
6
+ gemma-q4-dflash-q8,code,2,60,0.7095,0.7055,0.6789,0.7401
7
+ gemma-q4-dflash-q8,code,3,60,0.5906,0.5775,0.5531,0.6284
8
+ gemma-q4-dflash-q8,chat,1,55,0.6511,0.6700,0.5908,0.7066
9
+ gemma-q4-dflash-q8,chat,2,55,0.4744,0.4390,0.4124,0.5378
10
+ gemma-q4-dflash-q8,chat,3,55,0.3221,0.2390,0.2609,0.3876
experiments/analysis/curves/acc_by_pos_curves-gemma-q4-mtp.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-mtp,math,1,60,0.9051,0.9120,0.8935,0.9166
3
+ gemma-q4-mtp,math,2,60,0.8125,0.8160,0.7920,0.8327
4
+ gemma-q4-mtp,math,3,60,0.7374,0.7520,0.7137,0.7598
5
+ gemma-q4-mtp,code,1,60,0.8863,0.8960,0.8700,0.9020
6
+ gemma-q4-mtp,code,2,60,0.7817,0.7775,0.7572,0.8054
7
+ gemma-q4-mtp,code,3,60,0.6927,0.6885,0.6627,0.7231
8
+ gemma-q4-mtp,chat,1,55,0.7378,0.7860,0.6822,0.7893
9
+ gemma-q4-mtp,chat,2,55,0.5864,0.5740,0.5274,0.6486
10
+ gemma-q4-mtp,chat,3,55,0.4803,0.4360,0.4184,0.5485
experiments/analysis/curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q5-dflash-f16,math,1,60,0.8929,0.9050,0.8811,0.9041
3
+ gemma-q5-dflash-f16,math,2,60,0.7866,0.7970,0.7673,0.8056
4
+ gemma-q5-dflash-f16,math,3,60,0.6725,0.6780,0.6515,0.6936
5
+ gemma-q5-dflash-f16,code,1,58,0.8493,0.8710,0.8153,0.8769
6
+ gemma-q5-dflash-f16,code,2,58,0.7110,0.7265,0.6739,0.7490
7
+ gemma-q5-dflash-f16,code,3,58,0.5873,0.5820,0.5447,0.6348
8
+ gemma-q5-dflash-f16,chat,1,56,0.6458,0.6975,0.5908,0.6954
9
+ gemma-q5-dflash-f16,chat,2,56,0.4307,0.4210,0.3778,0.4835
10
+ gemma-q5-dflash-f16,chat,3,56,0.2739,0.2405,0.2242,0.3253
experiments/analysis/curves/acc_by_pos_curves-gemma-q5-mtp.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q5-mtp,math,1,60,0.9158,0.9170,0.9052,0.9254
3
+ gemma-q5-mtp,math,2,60,0.8320,0.8380,0.8127,0.8498
4
+ gemma-q5-mtp,math,3,60,0.7506,0.7570,0.7292,0.7723
5
+ gemma-q5-mtp,code,1,58,0.8731,0.8885,0.8504,0.8939
6
+ gemma-q5-mtp,code,2,58,0.7701,0.7775,0.7439,0.7976
7
+ gemma-q5-mtp,code,3,58,0.6722,0.6670,0.6396,0.7067
8
+ gemma-q5-mtp,chat,1,56,0.7448,0.7380,0.7027,0.7800
9
+ gemma-q5-mtp,chat,2,56,0.5743,0.5720,0.5256,0.6192
10
+ gemma-q5-mtp,chat,3,56,0.4511,0.4480,0.3983,0.5046
experiments/analysis/curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q8-dflash-f16,math,1,60,0.8919,0.9045,0.8784,0.9040
3
+ gemma-q8-dflash-f16,math,2,60,0.7721,0.7930,0.7505,0.7939
4
+ gemma-q8-dflash-f16,math,3,60,0.6460,0.6535,0.6191,0.6729
5
+ gemma-q8-dflash-f16,code,1,58,0.8427,0.8595,0.8168,0.8670
6
+ gemma-q8-dflash-f16,code,2,58,0.7027,0.6790,0.6676,0.7374
7
+ gemma-q8-dflash-f16,code,3,58,0.5889,0.5605,0.5460,0.6324
8
+ gemma-q8-dflash-f16,chat,1,56,0.6705,0.6720,0.6234,0.7139
9
+ gemma-q8-dflash-f16,chat,2,56,0.4360,0.4165,0.3837,0.4903
10
+ gemma-q8-dflash-f16,chat,3,56,0.2905,0.2595,0.2419,0.3434
experiments/analysis/curves/acc_by_pos_curves-gemma-q8-mtp.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q8-mtp,math,1,60,0.9139,0.9285,0.9010,0.9263
3
+ gemma-q8-mtp,math,2,60,0.8342,0.8480,0.8141,0.8543
4
+ gemma-q8-mtp,math,3,60,0.7476,0.7625,0.7243,0.7714
5
+ gemma-q8-mtp,code,1,58,0.8692,0.8655,0.8525,0.8869
6
+ gemma-q8-mtp,code,2,58,0.7647,0.7455,0.7410,0.7911
7
+ gemma-q8-mtp,code,3,58,0.6648,0.6515,0.6358,0.6977
8
+ gemma-q8-mtp,chat,1,56,0.7583,0.7555,0.7149,0.7956
9
+ gemma-q8-mtp,chat,2,56,0.5967,0.5870,0.5498,0.6417
10
+ gemma-q8-mtp,chat,3,56,0.4819,0.4470,0.4278,0.5349
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dflash.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dflash-f16,math,1,60,0.0200,0.0160,0.0160,0.0238
3
+ qwen-q4-dflash-f16,math,2,60,0.0048,0.0040,0.0032,0.0069
4
+ qwen-q4-dflash-f16,math,3,60,0.0018,0.0000,0.0011,0.0025
5
+ qwen-q4-dflash-f16,code,1,60,0.0215,0.0200,0.0182,0.0248
6
+ qwen-q4-dflash-f16,code,2,60,0.0043,0.0040,0.0033,0.0054
7
+ qwen-q4-dflash-f16,code,3,60,0.0018,0.0000,0.0011,0.0025
8
+ qwen-q4-dflash-f16,chat,1,58,0.0284,0.0205,0.0233,0.0338
9
+ qwen-q4-dflash-f16,chat,2,58,0.0042,0.0040,0.0032,0.0053
10
+ qwen-q4-dflash-f16,chat,3,58,0.0016,0.0000,0.0010,0.0022
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p0,math,1,60,0.8347,0.8510,0.8121,0.8570
3
+ qwen-q4-dspark-p0,math,2,60,0.6921,0.6975,0.6581,0.7273
4
+ qwen-q4-dspark-p0,math,3,60,0.5783,0.5645,0.5370,0.6213
5
+ qwen-q4-dspark-p0,code,1,60,0.7783,0.7705,0.7597,0.7969
6
+ qwen-q4-dspark-p0,code,2,60,0.6005,0.5930,0.5761,0.6254
7
+ qwen-q4-dspark-p0,code,3,60,0.4507,0.4320,0.4226,0.4816
8
+ qwen-q4-dspark-p0,chat,1,58,0.6920,0.7335,0.6444,0.7339
9
+ qwen-q4-dspark-p0,chat,2,58,0.4769,0.4975,0.4254,0.5239
10
+ qwen-q4-dspark-p0,chat,3,58,0.3412,0.3415,0.2928,0.3871
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p2,math,1,60,0.8345,0.8510,0.8120,0.8567
3
+ qwen-q4-dspark-p2,math,2,60,0.6918,0.6975,0.6576,0.7270
4
+ qwen-q4-dspark-p2,math,3,60,0.5783,0.5645,0.5370,0.6215
5
+ qwen-q4-dspark-p2,code,1,60,0.7778,0.7705,0.7592,0.7964
6
+ qwen-q4-dspark-p2,code,2,60,0.6003,0.5910,0.5758,0.6250
7
+ qwen-q4-dspark-p2,code,3,60,0.4508,0.4320,0.4225,0.4818
8
+ qwen-q4-dspark-p2,chat,1,58,0.6981,0.7405,0.6547,0.7378
9
+ qwen-q4-dspark-p2,chat,2,58,0.4825,0.5025,0.4326,0.5286
10
+ qwen-q4-dspark-p2,chat,3,58,0.3469,0.3605,0.2989,0.3935
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p4,math,1,60,0.8353,0.8520,0.8121,0.8580
3
+ qwen-q4-dspark-p4,math,2,60,0.6887,0.7000,0.6534,0.7253
4
+ qwen-q4-dspark-p4,math,3,60,0.5757,0.5560,0.5340,0.6201
5
+ qwen-q4-dspark-p4,code,1,60,0.7806,0.7890,0.7626,0.7979
6
+ qwen-q4-dspark-p4,code,2,60,0.6029,0.5945,0.5797,0.6280
7
+ qwen-q4-dspark-p4,code,3,60,0.4496,0.4325,0.4219,0.4785
8
+ qwen-q4-dspark-p4,chat,1,58,0.7250,0.7370,0.6935,0.7549
9
+ qwen-q4-dspark-p4,chat,2,58,0.4815,0.4800,0.4334,0.5258
10
+ qwen-q4-dspark-p4,chat,3,58,0.3297,0.3185,0.2819,0.3738
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p6,math,1,60,0.8438,0.8585,0.8235,0.8640
3
+ qwen-q4-dspark-p6,math,2,60,0.6839,0.6960,0.6493,0.7189
4
+ qwen-q4-dspark-p6,math,3,60,0.5639,0.5430,0.5213,0.6084
5
+ qwen-q4-dspark-p6,code,1,60,0.7965,0.7920,0.7783,0.8139
6
+ qwen-q4-dspark-p6,code,2,60,0.5830,0.5850,0.5583,0.6082
7
+ qwen-q4-dspark-p6,code,3,60,0.4117,0.4155,0.3843,0.4407
8
+ qwen-q4-dspark-p6,chat,1,58,0.7730,0.7805,0.7448,0.7983
9
+ qwen-q4-dspark-p6,chat,2,58,0.4486,0.4460,0.4033,0.4906
10
+ qwen-q4-dspark-p6,chat,3,58,0.2883,0.2700,0.2423,0.3329
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-eagle3.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-eagle3,math,1,60,0.6590,0.6410,0.6399,0.6790
3
+ qwen-q4-eagle3,math,2,60,0.4375,0.4340,0.4135,0.4620
4
+ qwen-q4-eagle3,math,3,60,0.2376,0.2375,0.2184,0.2586
5
+ qwen-q4-eagle3,code,1,60,0.6724,0.6715,0.6527,0.6925
6
+ qwen-q4-eagle3,code,2,60,0.4320,0.4195,0.4065,0.4582
7
+ qwen-q4-eagle3,code,3,60,0.2209,0.1965,0.1932,0.2509
8
+ qwen-q4-eagle3,chat,1,58,0.5734,0.6125,0.5174,0.6248
9
+ qwen-q4-eagle3,chat,2,58,0.3631,0.3525,0.3077,0.4165
10
+ qwen-q4-eagle3,chat,3,58,0.2106,0.1650,0.1654,0.2585
experiments/analysis/curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-vanilla17b,math,1,60,0.8725,0.8700,0.8594,0.8849
3
+ qwen-q4-vanilla17b,math,2,60,0.7706,0.7760,0.7498,0.7904
4
+ qwen-q4-vanilla17b,math,3,60,0.6855,0.6905,0.6592,0.7101
5
+ qwen-q4-vanilla17b,code,1,60,0.8352,0.8290,0.8234,0.8467
6
+ qwen-q4-vanilla17b,code,2,60,0.7095,0.7055,0.6907,0.7287
7
+ qwen-q4-vanilla17b,code,3,60,0.6087,0.6065,0.5858,0.6330
8
+ qwen-q4-vanilla17b,chat,1,58,0.7829,0.7745,0.7623,0.8032
9
+ qwen-q4-vanilla17b,chat,2,58,0.6254,0.6080,0.5945,0.6550
10
+ qwen-q4-vanilla17b,chat,3,58,0.5064,0.4880,0.4686,0.5435
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dflash.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q5-dflash-f16,math,1,60,0.0197,0.0160,0.0161,0.0233
3
+ qwen-q5-dflash-f16,math,2,60,0.0045,0.0040,0.0032,0.0059
4
+ qwen-q5-dflash-f16,math,3,60,0.0022,0.0000,0.0015,0.0031
5
+ qwen-q5-dflash-f16,code,1,60,0.0203,0.0200,0.0174,0.0233
6
+ qwen-q5-dflash-f16,code,2,60,0.0048,0.0040,0.0034,0.0063
7
+ qwen-q5-dflash-f16,code,3,60,0.0019,0.0000,0.0012,0.0026
8
+ qwen-q5-dflash-f16,chat,1,58,0.0264,0.0240,0.0217,0.0318
9
+ qwen-q5-dflash-f16,chat,2,58,0.0035,0.0040,0.0026,0.0044
10
+ qwen-q5-dflash-f16,chat,3,58,0.0013,0.0000,0.0008,0.0019
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q5-dspark-p0,math,1,60,0.8289,0.8320,0.8069,0.8512
3
+ qwen-q5-dspark-p0,math,2,60,0.6885,0.6895,0.6549,0.7223
4
+ qwen-q5-dspark-p0,math,3,60,0.5804,0.5835,0.5396,0.6222
5
+ qwen-q5-dspark-p0,code,1,60,0.7709,0.7685,0.7531,0.7896
6
+ qwen-q5-dspark-p0,code,2,60,0.5833,0.5815,0.5557,0.6125
7
+ qwen-q5-dspark-p0,code,3,60,0.4356,0.4285,0.4027,0.4701
8
+ qwen-q5-dspark-p0,chat,1,58,0.7017,0.7355,0.6575,0.7423
9
+ qwen-q5-dspark-p0,chat,2,58,0.4864,0.4925,0.4364,0.5348
10
+ qwen-q5-dspark-p0,chat,3,58,0.3534,0.3285,0.3056,0.4057
experiments/analysis/curves/acc_by_pos_curves-qwen-q5-eagle3.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q5-eagle3,math,1,60,0.6618,0.6515,0.6443,0.6798
3
+ qwen-q5-eagle3,math,2,60,0.4344,0.4290,0.4129,0.4560
4
+ qwen-q5-eagle3,math,3,60,0.2333,0.2145,0.2162,0.2506
5
+ qwen-q5-eagle3,code,1,60,0.6694,0.6685,0.6487,0.6908
6
+ qwen-q5-eagle3,code,2,60,0.4253,0.4245,0.3992,0.4533
7
+ qwen-q5-eagle3,code,3,60,0.2182,0.1960,0.1913,0.2485
8
+ qwen-q5-eagle3,chat,1,58,0.5931,0.6385,0.5418,0.6432
9
+ qwen-q5-eagle3,chat,2,58,0.3617,0.3620,0.3138,0.4103
10
+ qwen-q5-eagle3,chat,3,58,0.2002,0.1630,0.1611,0.2424
experiments/analysis/curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q8-dspark-p0,math,1,60,0.8264,0.8365,0.8037,0.8494
3
+ qwen-q8-dspark-p0,math,2,60,0.6872,0.6920,0.6529,0.7225
4
+ qwen-q8-dspark-p0,math,3,60,0.5736,0.5815,0.5309,0.6169
5
+ qwen-q8-dspark-p0,code,1,60,0.7736,0.7720,0.7569,0.7909
6
+ qwen-q8-dspark-p0,code,2,60,0.5855,0.5790,0.5615,0.6116
7
+ qwen-q8-dspark-p0,code,3,60,0.4410,0.4405,0.4163,0.4681
8
+ qwen-q8-dspark-p0,chat,1,58,0.6960,0.7210,0.6522,0.7347
9
+ qwen-q8-dspark-p0,chat,2,58,0.4866,0.4830,0.4410,0.5306
10
+ qwen-q8-dspark-p0,chat,3,58,0.3454,0.3235,0.3001,0.3922
experiments/analysis/curves/acc_by_pos_curves-qwen-q8-eagle3.csv ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q8-eagle3,math,1,60,0.6504,0.6445,0.6335,0.6669
3
+ qwen-q8-eagle3,math,2,60,0.4285,0.4180,0.4102,0.4492
4
+ qwen-q8-eagle3,math,3,60,0.2302,0.2345,0.2142,0.2476
5
+ qwen-q8-eagle3,code,1,60,0.6651,0.6535,0.6473,0.6835
6
+ qwen-q8-eagle3,code,2,60,0.4249,0.4165,0.4029,0.4487
7
+ qwen-q8-eagle3,code,3,60,0.2141,0.1890,0.1900,0.2404
8
+ qwen-q8-eagle3,chat,1,58,0.5834,0.6370,0.5311,0.6310
9
+ qwen-q8-eagle3,chat,2,58,0.3571,0.3720,0.3112,0.4011
10
+ qwen-q8-eagle3,chat,3,58,0.1905,0.1800,0.1568,0.2269
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-f16,math,1,60,0.8581,0.8600,0.8396,0.8743
3
+ gemma-q4-dflash-f16,math,2,60,0.7291,0.7255,0.7029,0.7541
4
+ gemma-q4-dflash-f16,math,3,60,0.5859,0.5935,0.5582,0.6130
5
+ gemma-q4-dflash-f16,math,4,60,0.4880,0.4710,0.4562,0.5191
6
+ gemma-q4-dflash-f16,math,5,60,0.3957,0.3750,0.3630,0.4270
7
+ gemma-q4-dflash-f16,math,6,60,0.3183,0.3030,0.2860,0.3491
8
+ gemma-q4-dflash-f16,math,7,60,0.2572,0.2255,0.2282,0.2844
9
+ gemma-q4-dflash-f16,math,8,60,0.2047,0.1840,0.1778,0.2304
10
+ gemma-q4-dflash-f16,math,9,60,0.1634,0.1615,0.1402,0.1858
11
+ gemma-q4-dflash-f16,math,10,60,0.1190,0.1020,0.0994,0.1383
12
+ gemma-q4-dflash-f16,code,1,60,0.7976,0.8050,0.7639,0.8275
13
+ gemma-q4-dflash-f16,code,2,60,0.6295,0.6485,0.5894,0.6649
14
+ gemma-q4-dflash-f16,code,3,60,0.4984,0.4920,0.4579,0.5403
15
+ gemma-q4-dflash-f16,code,4,60,0.3998,0.3985,0.3604,0.4414
16
+ gemma-q4-dflash-f16,code,5,60,0.3186,0.2845,0.2803,0.3592
17
+ gemma-q4-dflash-f16,code,6,60,0.2594,0.2250,0.2256,0.2962
18
+ gemma-q4-dflash-f16,code,7,60,0.2062,0.1760,0.1759,0.2401
19
+ gemma-q4-dflash-f16,code,8,60,0.1638,0.1355,0.1355,0.1968
20
+ gemma-q4-dflash-f16,code,9,60,0.1324,0.0935,0.1053,0.1640
21
+ gemma-q4-dflash-f16,code,10,60,0.1047,0.0765,0.0809,0.1346
22
+ gemma-q4-dflash-f16,chat,1,55,0.6401,0.6670,0.5908,0.6840
23
+ gemma-q4-dflash-f16,chat,2,55,0.4230,0.4220,0.3681,0.4749
24
+ gemma-q4-dflash-f16,chat,3,55,0.2565,0.2060,0.2103,0.3053
25
+ gemma-q4-dflash-f16,chat,4,55,0.1807,0.1370,0.1361,0.2264
26
+ gemma-q4-dflash-f16,chat,5,55,0.1324,0.0820,0.0922,0.1771
27
+ gemma-q4-dflash-f16,chat,6,55,0.0946,0.0410,0.0585,0.1332
28
+ gemma-q4-dflash-f16,chat,7,55,0.0712,0.0200,0.0384,0.1063
29
+ gemma-q4-dflash-f16,chat,8,55,0.0589,0.0110,0.0292,0.0933
30
+ gemma-q4-dflash-f16,chat,9,55,0.0447,0.0090,0.0187,0.0754
31
+ gemma-q4-dflash-f16,chat,10,55,0.0355,0.0000,0.0134,0.0626
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-f16,math,1,60,0.8648,0.8805,0.8466,0.8817
3
+ gemma-q4-dflash-f16,math,2,60,0.7476,0.7540,0.7238,0.7708
4
+ gemma-q4-dflash-f16,math,3,60,0.6149,0.6180,0.5844,0.6425
5
+ gemma-q4-dflash-f16,math,4,60,0.5182,0.5040,0.4846,0.5506
6
+ gemma-q4-dflash-f16,math,5,60,0.4348,0.4370,0.4013,0.4672
7
+ gemma-q4-dflash-f16,code,1,60,0.8291,0.8310,0.8066,0.8516
8
+ gemma-q4-dflash-f16,code,2,60,0.6822,0.6800,0.6523,0.7136
9
+ gemma-q4-dflash-f16,code,3,60,0.5617,0.5665,0.5240,0.5997
10
+ gemma-q4-dflash-f16,code,4,60,0.4657,0.4620,0.4247,0.5074
11
+ gemma-q4-dflash-f16,code,5,60,0.3904,0.3770,0.3486,0.4327
12
+ gemma-q4-dflash-f16,chat,1,55,0.6564,0.6800,0.6046,0.7042
13
+ gemma-q4-dflash-f16,chat,2,55,0.4466,0.4130,0.3915,0.5006
14
+ gemma-q4-dflash-f16,chat,3,55,0.2992,0.2500,0.2453,0.3531
15
+ gemma-q4-dflash-f16,chat,4,55,0.2237,0.1490,0.1725,0.2766
16
+ gemma-q4-dflash-f16,chat,5,55,0.1691,0.1060,0.1185,0.2226
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-dflash-f16,math,1,60,0.8579,0.8670,0.8395,0.8751
3
+ gemma-q4-dflash-f16,math,2,60,0.7314,0.7405,0.7069,0.7555
4
+ gemma-q4-dflash-f16,math,3,60,0.5956,0.6050,0.5641,0.6247
5
+ gemma-q4-dflash-f16,math,4,60,0.4932,0.4820,0.4577,0.5276
6
+ gemma-q4-dflash-f16,math,5,60,0.4169,0.3910,0.3822,0.4509
7
+ gemma-q4-dflash-f16,math,6,60,0.3513,0.3330,0.3177,0.3852
8
+ gemma-q4-dflash-f16,math,7,60,0.2898,0.2665,0.2592,0.3210
9
+ gemma-q4-dflash-f16,code,1,60,0.8238,0.8230,0.7995,0.8470
10
+ gemma-q4-dflash-f16,code,2,60,0.6661,0.6670,0.6331,0.6983
11
+ gemma-q4-dflash-f16,code,3,60,0.5352,0.5340,0.4997,0.5731
12
+ gemma-q4-dflash-f16,code,4,60,0.4426,0.4410,0.4046,0.4808
13
+ gemma-q4-dflash-f16,code,5,60,0.3694,0.3695,0.3308,0.4081
14
+ gemma-q4-dflash-f16,code,6,60,0.3013,0.2845,0.2641,0.3397
15
+ gemma-q4-dflash-f16,code,7,60,0.2465,0.2235,0.2120,0.2846
16
+ gemma-q4-dflash-f16,chat,1,55,0.6426,0.6580,0.5935,0.6863
17
+ gemma-q4-dflash-f16,chat,2,55,0.4293,0.4090,0.3773,0.4805
18
+ gemma-q4-dflash-f16,chat,3,55,0.2806,0.2420,0.2307,0.3323
19
+ gemma-q4-dflash-f16,chat,4,55,0.2043,0.1510,0.1562,0.2537
20
+ gemma-q4-dflash-f16,chat,5,55,0.1530,0.0870,0.1073,0.2019
21
+ gemma-q4-dflash-f16,chat,6,55,0.1202,0.0510,0.0773,0.1675
22
+ gemma-q4-dflash-f16,chat,7,55,0.0942,0.0280,0.0528,0.1403
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-mtp,math,1,60,0.8837,0.8905,0.8682,0.8992
3
+ gemma-q4-mtp,math,2,60,0.7941,0.8000,0.7706,0.8170
4
+ gemma-q4-mtp,math,3,60,0.7048,0.7195,0.6784,0.7300
5
+ gemma-q4-mtp,math,4,60,0.6123,0.6385,0.5747,0.6443
6
+ gemma-q4-mtp,math,5,60,0.5342,0.5490,0.4978,0.5670
7
+ gemma-q4-mtp,math,6,60,0.4595,0.4630,0.4243,0.4929
8
+ gemma-q4-mtp,math,7,60,0.3834,0.3735,0.3502,0.4155
9
+ gemma-q4-mtp,math,8,60,0.3149,0.3115,0.2849,0.3450
10
+ gemma-q4-mtp,math,9,60,0.2509,0.2415,0.2232,0.2811
11
+ gemma-q4-mtp,math,10,60,0.2024,0.2000,0.1794,0.2282
12
+ gemma-q4-mtp,code,1,60,0.8295,0.8435,0.8010,0.8528
13
+ gemma-q4-mtp,code,2,60,0.7212,0.7320,0.6897,0.7507
14
+ gemma-q4-mtp,code,3,60,0.6143,0.6040,0.5809,0.6463
15
+ gemma-q4-mtp,code,4,60,0.5351,0.5230,0.5003,0.5712
16
+ gemma-q4-mtp,code,5,60,0.4681,0.4495,0.4327,0.5033
17
+ gemma-q4-mtp,code,6,60,0.3917,0.3770,0.3582,0.4263
18
+ gemma-q4-mtp,code,7,60,0.3331,0.3090,0.2998,0.3674
19
+ gemma-q4-mtp,code,8,60,0.2793,0.2500,0.2477,0.3137
20
+ gemma-q4-mtp,code,9,60,0.2336,0.2140,0.2039,0.2662
21
+ gemma-q4-mtp,code,10,60,0.1817,0.1590,0.1529,0.2141
22
+ gemma-q4-mtp,chat,1,55,0.7338,0.7470,0.6891,0.7732
23
+ gemma-q4-mtp,chat,2,55,0.5758,0.5690,0.5250,0.6269
24
+ gemma-q4-mtp,chat,3,55,0.4411,0.3880,0.3849,0.5014
25
+ gemma-q4-mtp,chat,4,55,0.3483,0.2770,0.2882,0.4102
26
+ gemma-q4-mtp,chat,5,55,0.2700,0.1920,0.2119,0.3302
27
+ gemma-q4-mtp,chat,6,55,0.2250,0.1390,0.1672,0.2859
28
+ gemma-q4-mtp,chat,7,55,0.1913,0.0920,0.1341,0.2554
29
+ gemma-q4-mtp,chat,8,55,0.1640,0.0690,0.1064,0.2285
30
+ gemma-q4-mtp,chat,9,55,0.1394,0.0430,0.0820,0.2036
31
+ gemma-q4-mtp,chat,10,55,0.1244,0.0280,0.0660,0.1884
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-mtp,math,1,60,0.8950,0.8975,0.8799,0.9086
3
+ gemma-q4-mtp,math,2,60,0.8062,0.8085,0.7859,0.8247
4
+ gemma-q4-mtp,math,3,60,0.7245,0.7430,0.6978,0.7487
5
+ gemma-q4-mtp,math,4,60,0.6364,0.6530,0.6067,0.6637
6
+ gemma-q4-mtp,math,5,60,0.5608,0.5600,0.5306,0.5901
7
+ gemma-q4-mtp,code,1,60,0.8678,0.8850,0.8489,0.8859
8
+ gemma-q4-mtp,code,2,60,0.7681,0.7740,0.7418,0.7935
9
+ gemma-q4-mtp,code,3,60,0.6668,0.6610,0.6358,0.6977
10
+ gemma-q4-mtp,code,4,60,0.5806,0.5680,0.5440,0.6167
11
+ gemma-q4-mtp,code,5,60,0.5110,0.5000,0.4725,0.5504
12
+ gemma-q4-mtp,chat,1,55,0.7516,0.7570,0.7078,0.7899
13
+ gemma-q4-mtp,chat,2,55,0.5933,0.5850,0.5418,0.6487
14
+ gemma-q4-mtp,chat,3,55,0.4816,0.4120,0.4221,0.5442
15
+ gemma-q4-mtp,chat,4,55,0.3951,0.3250,0.3306,0.4627
16
+ gemma-q4-mtp,chat,5,55,0.3237,0.2190,0.2579,0.3945
experiments/analysis/curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ gemma-q4-mtp,math,1,60,0.8807,0.8830,0.8625,0.8965
3
+ gemma-q4-mtp,math,2,60,0.7945,0.8025,0.7706,0.8170
4
+ gemma-q4-mtp,math,3,60,0.7120,0.7085,0.6852,0.7385
5
+ gemma-q4-mtp,math,4,60,0.6326,0.6380,0.6015,0.6633
6
+ gemma-q4-mtp,math,5,60,0.5558,0.5530,0.5223,0.5895
7
+ gemma-q4-mtp,math,6,60,0.4760,0.4825,0.4397,0.5133
8
+ gemma-q4-mtp,math,7,60,0.4050,0.3900,0.3713,0.4385
9
+ gemma-q4-mtp,code,1,60,0.8569,0.8740,0.8370,0.8764
10
+ gemma-q4-mtp,code,2,60,0.7463,0.7630,0.7187,0.7722
11
+ gemma-q4-mtp,code,3,60,0.6522,0.6540,0.6200,0.6846
12
+ gemma-q4-mtp,code,4,60,0.5666,0.5680,0.5307,0.6020
13
+ gemma-q4-mtp,code,5,60,0.4945,0.4755,0.4555,0.5327
14
+ gemma-q4-mtp,code,6,60,0.4205,0.3970,0.3808,0.4608
15
+ gemma-q4-mtp,code,7,60,0.3630,0.3365,0.3227,0.4041
16
+ gemma-q4-mtp,chat,1,55,0.7442,0.7500,0.7010,0.7816
17
+ gemma-q4-mtp,chat,2,55,0.5852,0.5610,0.5343,0.6413
18
+ gemma-q4-mtp,chat,3,55,0.4759,0.4110,0.4179,0.5385
19
+ gemma-q4-mtp,chat,4,55,0.3938,0.3010,0.3297,0.4655
20
+ gemma-q4-mtp,chat,5,55,0.3201,0.2030,0.2554,0.3891
21
+ gemma-q4-mtp,chat,6,55,0.2712,0.1590,0.2051,0.3428
22
+ gemma-q4-mtp,chat,7,55,0.2360,0.1130,0.1668,0.3091
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dflash-f16,math,1,60,0.0147,0.0100,0.0113,0.0186
3
+ qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0018,0.0056
4
+ qwen-q4-dflash-f16,math,3,60,0.0017,0.0000,0.0009,0.0025
5
+ qwen-q4-dflash-f16,math,4,60,0.0011,0.0000,0.0007,0.0017
6
+ qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
7
+ qwen-q4-dflash-f16,math,6,60,0.0003,0.0000,0.0001,0.0007
8
+ qwen-q4-dflash-f16,math,7,60,0.0000,0.0000,0.0000,0.0000
9
+ qwen-q4-dflash-f16,math,8,60,0.0000,0.0000,0.0000,0.0000
10
+ qwen-q4-dflash-f16,math,9,60,0.0000,0.0000,0.0000,0.0000
11
+ qwen-q4-dflash-f16,math,10,60,0.0000,0.0000,0.0000,0.0000
12
+ qwen-q4-dflash-f16,code,1,60,0.0138,0.0120,0.0112,0.0165
13
+ qwen-q4-dflash-f16,code,2,60,0.0039,0.0040,0.0028,0.0052
14
+ qwen-q4-dflash-f16,code,3,60,0.0015,0.0000,0.0009,0.0024
15
+ qwen-q4-dflash-f16,code,4,60,0.0006,0.0000,0.0002,0.0011
16
+ qwen-q4-dflash-f16,code,5,60,0.0003,0.0000,0.0001,0.0007
17
+ qwen-q4-dflash-f16,code,6,60,0.0002,0.0000,0.0000,0.0005
18
+ qwen-q4-dflash-f16,code,7,60,0.0000,0.0000,0.0000,0.0000
19
+ qwen-q4-dflash-f16,code,8,60,0.0000,0.0000,0.0000,0.0000
20
+ qwen-q4-dflash-f16,code,9,60,0.0000,0.0000,0.0000,0.0000
21
+ qwen-q4-dflash-f16,code,10,60,0.0000,0.0000,0.0000,0.0000
22
+ qwen-q4-dflash-f16,chat,1,58,0.0214,0.0160,0.0171,0.0264
23
+ qwen-q4-dflash-f16,chat,2,58,0.0046,0.0040,0.0033,0.0060
24
+ qwen-q4-dflash-f16,chat,3,58,0.0019,0.0000,0.0011,0.0027
25
+ qwen-q4-dflash-f16,chat,4,58,0.0010,0.0000,0.0004,0.0017
26
+ qwen-q4-dflash-f16,chat,5,58,0.0005,0.0000,0.0001,0.0011
27
+ qwen-q4-dflash-f16,chat,6,58,0.0003,0.0000,0.0000,0.0007
28
+ qwen-q4-dflash-f16,chat,7,58,0.0000,0.0000,0.0000,0.0000
29
+ qwen-q4-dflash-f16,chat,8,58,0.0000,0.0000,0.0000,0.0000
30
+ qwen-q4-dflash-f16,chat,9,58,0.0000,0.0000,0.0000,0.0000
31
+ qwen-q4-dflash-f16,chat,10,58,0.0000,0.0000,0.0000,0.0000
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dflash-f16,math,1,60,0.0139,0.0120,0.0105,0.0178
3
+ qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0019,0.0056
4
+ qwen-q4-dflash-f16,math,3,60,0.0013,0.0000,0.0007,0.0019
5
+ qwen-q4-dflash-f16,math,4,60,0.0008,0.0000,0.0004,0.0012
6
+ qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
7
+ qwen-q4-dflash-f16,code,1,60,0.0126,0.0120,0.0101,0.0152
8
+ qwen-q4-dflash-f16,code,2,60,0.0030,0.0000,0.0020,0.0040
9
+ qwen-q4-dflash-f16,code,3,60,0.0013,0.0000,0.0007,0.0018
10
+ qwen-q4-dflash-f16,code,4,60,0.0005,0.0000,0.0002,0.0009
11
+ qwen-q4-dflash-f16,code,5,60,0.0002,0.0000,0.0000,0.0005
12
+ qwen-q4-dflash-f16,chat,1,58,0.0209,0.0160,0.0166,0.0257
13
+ qwen-q4-dflash-f16,chat,2,58,0.0036,0.0000,0.0024,0.0051
14
+ qwen-q4-dflash-f16,chat,3,58,0.0012,0.0000,0.0007,0.0018
15
+ qwen-q4-dflash-f16,chat,4,58,0.0006,0.0000,0.0003,0.0010
16
+ qwen-q4-dflash-f16,chat,5,58,0.0002,0.0000,0.0000,0.0005
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dflash-f16,math,1,60,0.0147,0.0100,0.0113,0.0186
3
+ qwen-q4-dflash-f16,math,2,60,0.0034,0.0000,0.0018,0.0056
4
+ qwen-q4-dflash-f16,math,3,60,0.0017,0.0000,0.0009,0.0025
5
+ qwen-q4-dflash-f16,math,4,60,0.0011,0.0000,0.0007,0.0017
6
+ qwen-q4-dflash-f16,math,5,60,0.0006,0.0000,0.0003,0.0010
7
+ qwen-q4-dflash-f16,math,6,60,0.0003,0.0000,0.0001,0.0007
8
+ qwen-q4-dflash-f16,math,7,60,0.0000,0.0000,0.0000,0.0000
9
+ qwen-q4-dflash-f16,code,1,60,0.0138,0.0120,0.0112,0.0165
10
+ qwen-q4-dflash-f16,code,2,60,0.0039,0.0040,0.0028,0.0052
11
+ qwen-q4-dflash-f16,code,3,60,0.0015,0.0000,0.0009,0.0024
12
+ qwen-q4-dflash-f16,code,4,60,0.0006,0.0000,0.0002,0.0011
13
+ qwen-q4-dflash-f16,code,5,60,0.0003,0.0000,0.0001,0.0007
14
+ qwen-q4-dflash-f16,code,6,60,0.0002,0.0000,0.0000,0.0005
15
+ qwen-q4-dflash-f16,code,7,60,0.0000,0.0000,0.0000,0.0000
16
+ qwen-q4-dflash-f16,chat,1,58,0.0214,0.0160,0.0171,0.0264
17
+ qwen-q4-dflash-f16,chat,2,58,0.0046,0.0040,0.0033,0.0060
18
+ qwen-q4-dflash-f16,chat,3,58,0.0019,0.0000,0.0011,0.0027
19
+ qwen-q4-dflash-f16,chat,4,58,0.0010,0.0000,0.0004,0.0017
20
+ qwen-q4-dflash-f16,chat,5,58,0.0005,0.0000,0.0001,0.0011
21
+ qwen-q4-dflash-f16,chat,6,58,0.0003,0.0000,0.0000,0.0007
22
+ qwen-q4-dflash-f16,chat,7,58,0.0000,0.0000,0.0000,0.0000
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p0,math,1,60,0.8275,0.8235,0.8088,0.8465
3
+ qwen-q4-dspark-p0,math,2,60,0.6883,0.7065,0.6567,0.7224
4
+ qwen-q4-dspark-p0,math,3,60,0.5771,0.5700,0.5378,0.6177
5
+ qwen-q4-dspark-p0,math,4,60,0.4843,0.4585,0.4372,0.5337
6
+ qwen-q4-dspark-p0,math,5,60,0.4117,0.4000,0.3624,0.4656
7
+ qwen-q4-dspark-p0,math,6,60,0.3543,0.3390,0.3045,0.4083
8
+ qwen-q4-dspark-p0,math,7,60,0.3056,0.2950,0.2568,0.3597
9
+ qwen-q4-dspark-p0,math,8,60,0.0000,0.0000,0.0000,0.0000
10
+ qwen-q4-dspark-p0,math,9,60,0.0000,0.0000,0.0000,0.0000
11
+ qwen-q4-dspark-p0,math,10,60,0.0000,0.0000,0.0000,0.0000
12
+ qwen-q4-dspark-p0,code,1,60,0.7944,0.8080,0.7753,0.8127
13
+ qwen-q4-dspark-p0,code,2,60,0.6135,0.6125,0.5853,0.6421
14
+ qwen-q4-dspark-p0,code,3,60,0.4618,0.4355,0.4318,0.4931
15
+ qwen-q4-dspark-p0,code,4,60,0.3523,0.3255,0.3217,0.3855
16
+ qwen-q4-dspark-p0,code,5,60,0.2684,0.2355,0.2363,0.3034
17
+ qwen-q4-dspark-p0,code,6,60,0.1977,0.1725,0.1691,0.2283
18
+ qwen-q4-dspark-p0,code,7,60,0.1526,0.1190,0.1241,0.1840
19
+ qwen-q4-dspark-p0,code,8,60,0.0000,0.0000,0.0000,0.0000
20
+ qwen-q4-dspark-p0,code,9,60,0.0000,0.0000,0.0000,0.0000
21
+ qwen-q4-dspark-p0,code,10,60,0.0000,0.0000,0.0000,0.0000
22
+ qwen-q4-dspark-p0,chat,1,58,0.7089,0.7275,0.6651,0.7472
23
+ qwen-q4-dspark-p0,chat,2,58,0.4922,0.5095,0.4431,0.5365
24
+ qwen-q4-dspark-p0,chat,3,58,0.3525,0.3375,0.3044,0.3973
25
+ qwen-q4-dspark-p0,chat,4,58,0.2589,0.2205,0.2158,0.3017
26
+ qwen-q4-dspark-p0,chat,5,58,0.1988,0.1545,0.1588,0.2404
27
+ qwen-q4-dspark-p0,chat,6,58,0.1552,0.0895,0.1173,0.1929
28
+ qwen-q4-dspark-p0,chat,7,58,0.1229,0.0650,0.0873,0.1596
29
+ qwen-q4-dspark-p0,chat,8,58,0.0000,0.0000,0.0000,0.0000
30
+ qwen-q4-dspark-p0,chat,9,58,0.0000,0.0000,0.0000,0.0000
31
+ qwen-q4-dspark-p0,chat,10,58,0.0000,0.0000,0.0000,0.0000
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p0,math,1,60,0.8351,0.8435,0.8159,0.8546
3
+ qwen-q4-dspark-p0,math,2,60,0.6908,0.6990,0.6574,0.7251
4
+ qwen-q4-dspark-p0,math,3,60,0.5831,0.5715,0.5408,0.6269
5
+ qwen-q4-dspark-p0,math,4,60,0.4884,0.4880,0.4413,0.5375
6
+ qwen-q4-dspark-p0,math,5,60,0.4097,0.3970,0.3609,0.4619
7
+ qwen-q4-dspark-p0,code,1,60,0.7904,0.7920,0.7718,0.8086
8
+ qwen-q4-dspark-p0,code,2,60,0.6033,0.5945,0.5752,0.6310
9
+ qwen-q4-dspark-p0,code,3,60,0.4645,0.4540,0.4331,0.4965
10
+ qwen-q4-dspark-p0,code,4,60,0.3539,0.3290,0.3204,0.3897
11
+ qwen-q4-dspark-p0,code,5,60,0.2637,0.2300,0.2305,0.2998
12
+ qwen-q4-dspark-p0,chat,1,58,0.7086,0.7555,0.6648,0.7460
13
+ qwen-q4-dspark-p0,chat,2,58,0.4859,0.5130,0.4345,0.5313
14
+ qwen-q4-dspark-p0,chat,3,58,0.3529,0.3545,0.3042,0.3996
15
+ qwen-q4-dspark-p0,chat,4,58,0.2674,0.2155,0.2195,0.3135
16
+ qwen-q4-dspark-p0,chat,5,58,0.2048,0.1445,0.1607,0.2485
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-dspark-p0,math,1,60,0.8275,0.8235,0.8088,0.8465
3
+ qwen-q4-dspark-p0,math,2,60,0.6883,0.7065,0.6567,0.7224
4
+ qwen-q4-dspark-p0,math,3,60,0.5771,0.5700,0.5378,0.6177
5
+ qwen-q4-dspark-p0,math,4,60,0.4843,0.4585,0.4372,0.5337
6
+ qwen-q4-dspark-p0,math,5,60,0.4117,0.4000,0.3624,0.4656
7
+ qwen-q4-dspark-p0,math,6,60,0.3543,0.3390,0.3045,0.4083
8
+ qwen-q4-dspark-p0,math,7,60,0.3056,0.2950,0.2568,0.3597
9
+ qwen-q4-dspark-p0,code,1,60,0.7944,0.8080,0.7753,0.8127
10
+ qwen-q4-dspark-p0,code,2,60,0.6135,0.6125,0.5853,0.6421
11
+ qwen-q4-dspark-p0,code,3,60,0.4618,0.4355,0.4318,0.4931
12
+ qwen-q4-dspark-p0,code,4,60,0.3523,0.3255,0.3217,0.3855
13
+ qwen-q4-dspark-p0,code,5,60,0.2684,0.2355,0.2363,0.3034
14
+ qwen-q4-dspark-p0,code,6,60,0.1977,0.1725,0.1691,0.2283
15
+ qwen-q4-dspark-p0,code,7,60,0.1526,0.1190,0.1241,0.1840
16
+ qwen-q4-dspark-p0,chat,1,58,0.7089,0.7275,0.6651,0.7472
17
+ qwen-q4-dspark-p0,chat,2,58,0.4922,0.5095,0.4431,0.5365
18
+ qwen-q4-dspark-p0,chat,3,58,0.3525,0.3375,0.3044,0.3973
19
+ qwen-q4-dspark-p0,chat,4,58,0.2589,0.2205,0.2158,0.3017
20
+ qwen-q4-dspark-p0,chat,5,58,0.1988,0.1545,0.1588,0.2404
21
+ qwen-q4-dspark-p0,chat,6,58,0.1552,0.0895,0.1173,0.1929
22
+ qwen-q4-dspark-p0,chat,7,58,0.1229,0.0650,0.0873,0.1596
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-eagle3,math,1,60,0.6393,0.6360,0.6221,0.6572
3
+ qwen-q4-eagle3,math,2,60,0.4121,0.4130,0.3912,0.4337
4
+ qwen-q4-eagle3,math,3,60,0.2157,0.2145,0.2011,0.2309
5
+ qwen-q4-eagle3,math,4,60,0.1215,0.1165,0.1095,0.1351
6
+ qwen-q4-eagle3,math,5,60,0.0731,0.0705,0.0631,0.0844
7
+ qwen-q4-eagle3,math,6,60,0.0449,0.0380,0.0372,0.0539
8
+ qwen-q4-eagle3,math,7,60,0.0274,0.0225,0.0218,0.0335
9
+ qwen-q4-eagle3,math,8,60,0.0165,0.0110,0.0126,0.0207
10
+ qwen-q4-eagle3,math,9,60,0.0087,0.0085,0.0054,0.0124
11
+ qwen-q4-eagle3,math,10,60,0.0058,0.0000,0.0032,0.0086
12
+ qwen-q4-eagle3,code,1,60,0.6602,0.6415,0.6420,0.6794
13
+ qwen-q4-eagle3,code,2,60,0.4151,0.3990,0.3912,0.4416
14
+ qwen-q4-eagle3,code,3,60,0.2156,0.1740,0.1876,0.2477
15
+ qwen-q4-eagle3,code,4,60,0.1279,0.0925,0.1029,0.1570
16
+ qwen-q4-eagle3,code,5,60,0.0874,0.0460,0.0640,0.1149
17
+ qwen-q4-eagle3,code,6,60,0.0572,0.0200,0.0382,0.0792
18
+ qwen-q4-eagle3,code,7,60,0.0389,0.0100,0.0229,0.0571
19
+ qwen-q4-eagle3,code,8,60,0.0305,0.0090,0.0162,0.0474
20
+ qwen-q4-eagle3,code,9,60,0.0217,0.0000,0.0105,0.0353
21
+ qwen-q4-eagle3,code,10,60,0.0163,0.0000,0.0081,0.0259
22
+ qwen-q4-eagle3,chat,1,58,0.5736,0.6015,0.5236,0.6212
23
+ qwen-q4-eagle3,chat,2,58,0.3468,0.3715,0.2998,0.3917
24
+ qwen-q4-eagle3,chat,3,58,0.1888,0.1630,0.1524,0.2246
25
+ qwen-q4-eagle3,chat,4,58,0.1189,0.0790,0.0884,0.1512
26
+ qwen-q4-eagle3,chat,5,58,0.0831,0.0405,0.0580,0.1095
27
+ qwen-q4-eagle3,chat,6,58,0.0650,0.0285,0.0418,0.0898
28
+ qwen-q4-eagle3,chat,7,58,0.0490,0.0095,0.0291,0.0712
29
+ qwen-q4-eagle3,chat,8,58,0.0407,0.0080,0.0223,0.0618
30
+ qwen-q4-eagle3,chat,9,58,0.0344,0.0000,0.0168,0.0545
31
+ qwen-q4-eagle3,chat,10,58,0.0292,0.0000,0.0136,0.0473
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-eagle3,math,1,60,0.6475,0.6315,0.6282,0.6682
3
+ qwen-q4-eagle3,math,2,60,0.4171,0.4050,0.3944,0.4411
4
+ qwen-q4-eagle3,math,3,60,0.2238,0.2180,0.2056,0.2438
5
+ qwen-q4-eagle3,math,4,60,0.1287,0.1205,0.1123,0.1467
6
+ qwen-q4-eagle3,math,5,60,0.0775,0.0695,0.0650,0.0921
7
+ qwen-q4-eagle3,code,1,60,0.6593,0.6480,0.6400,0.6802
8
+ qwen-q4-eagle3,code,2,60,0.4212,0.4105,0.3942,0.4513
9
+ qwen-q4-eagle3,code,3,60,0.2141,0.1760,0.1854,0.2466
10
+ qwen-q4-eagle3,code,4,60,0.1281,0.0880,0.1010,0.1594
11
+ qwen-q4-eagle3,code,5,60,0.0828,0.0480,0.0581,0.1121
12
+ qwen-q4-eagle3,chat,1,58,0.5732,0.5970,0.5165,0.6252
13
+ qwen-q4-eagle3,chat,2,58,0.3544,0.3435,0.3023,0.4045
14
+ qwen-q4-eagle3,chat,3,58,0.1993,0.1605,0.1593,0.2427
15
+ qwen-q4-eagle3,chat,4,58,0.1258,0.0920,0.0925,0.1614
16
+ qwen-q4-eagle3,chat,5,58,0.0821,0.0435,0.0545,0.1116
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-eagle3,math,1,60,0.6443,0.6260,0.6251,0.6649
3
+ qwen-q4-eagle3,math,2,60,0.4131,0.4030,0.3907,0.4366
4
+ qwen-q4-eagle3,math,3,60,0.2196,0.2180,0.2024,0.2386
5
+ qwen-q4-eagle3,math,4,60,0.1262,0.1180,0.1105,0.1435
6
+ qwen-q4-eagle3,math,5,60,0.0755,0.0675,0.0633,0.0897
7
+ qwen-q4-eagle3,math,6,60,0.0456,0.0360,0.0356,0.0571
8
+ qwen-q4-eagle3,math,7,60,0.0280,0.0200,0.0204,0.0374
9
+ qwen-q4-eagle3,code,1,60,0.6557,0.6445,0.6375,0.6748
10
+ qwen-q4-eagle3,code,2,60,0.4176,0.4125,0.3919,0.4460
11
+ qwen-q4-eagle3,code,3,60,0.2125,0.1815,0.1843,0.2452
12
+ qwen-q4-eagle3,code,4,60,0.1263,0.0875,0.0983,0.1595
13
+ qwen-q4-eagle3,code,5,60,0.0813,0.0490,0.0565,0.1118
14
+ qwen-q4-eagle3,code,6,60,0.0526,0.0190,0.0320,0.0773
15
+ qwen-q4-eagle3,code,7,60,0.0377,0.0100,0.0201,0.0593
16
+ qwen-q4-eagle3,chat,1,58,0.5708,0.6045,0.5146,0.6221
17
+ qwen-q4-eagle3,chat,2,58,0.3497,0.3395,0.2993,0.3989
18
+ qwen-q4-eagle3,chat,3,58,0.1902,0.1580,0.1547,0.2273
19
+ qwen-q4-eagle3,chat,4,58,0.1203,0.0840,0.0906,0.1531
20
+ qwen-q4-eagle3,chat,5,58,0.0753,0.0425,0.0509,0.1010
21
+ qwen-q4-eagle3,chat,6,58,0.0579,0.0205,0.0352,0.0826
22
+ qwen-q4-eagle3,chat,7,58,0.0427,0.0090,0.0227,0.0639
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-vanilla17b,math,1,60,0.8606,0.8640,0.8454,0.8754
3
+ qwen-q4-vanilla17b,math,2,60,0.7416,0.7255,0.7152,0.7684
4
+ qwen-q4-vanilla17b,math,3,60,0.6567,0.6505,0.6250,0.6896
5
+ qwen-q4-vanilla17b,math,4,60,0.5859,0.5725,0.5521,0.6207
6
+ qwen-q4-vanilla17b,math,5,60,0.5293,0.5120,0.4932,0.5667
7
+ qwen-q4-vanilla17b,math,6,60,0.4771,0.4690,0.4406,0.5155
8
+ qwen-q4-vanilla17b,math,7,60,0.4277,0.4200,0.3902,0.4668
9
+ qwen-q4-vanilla17b,math,8,60,0.3844,0.3660,0.3468,0.4236
10
+ qwen-q4-vanilla17b,math,9,60,0.3481,0.3410,0.3101,0.3869
11
+ qwen-q4-vanilla17b,math,10,60,0.3208,0.3140,0.2823,0.3584
12
+ qwen-q4-vanilla17b,code,1,60,0.8284,0.8350,0.8096,0.8477
13
+ qwen-q4-vanilla17b,code,2,60,0.6865,0.6875,0.6631,0.7102
14
+ qwen-q4-vanilla17b,code,3,60,0.5856,0.5780,0.5595,0.6142
15
+ qwen-q4-vanilla17b,code,4,60,0.5046,0.4840,0.4784,0.5369
16
+ qwen-q4-vanilla17b,code,5,60,0.4359,0.4150,0.4082,0.4698
17
+ qwen-q4-vanilla17b,code,6,60,0.3831,0.3645,0.3531,0.4188
18
+ qwen-q4-vanilla17b,code,7,60,0.3410,0.3225,0.3108,0.3766
19
+ qwen-q4-vanilla17b,code,8,60,0.3017,0.2780,0.2722,0.3373
20
+ qwen-q4-vanilla17b,code,9,60,0.2707,0.2420,0.2401,0.3074
21
+ qwen-q4-vanilla17b,code,10,60,0.2394,0.2145,0.2116,0.2720
22
+ qwen-q4-vanilla17b,chat,1,58,0.7707,0.7445,0.7471,0.7945
23
+ qwen-q4-vanilla17b,chat,2,58,0.5909,0.5880,0.5608,0.6219
24
+ qwen-q4-vanilla17b,chat,3,58,0.4747,0.4780,0.4397,0.5127
25
+ qwen-q4-vanilla17b,chat,4,58,0.3836,0.3620,0.3472,0.4236
26
+ qwen-q4-vanilla17b,chat,5,58,0.3246,0.2845,0.2864,0.3674
27
+ qwen-q4-vanilla17b,chat,6,58,0.2791,0.2310,0.2390,0.3239
28
+ qwen-q4-vanilla17b,chat,7,58,0.2419,0.1910,0.2015,0.2864
29
+ qwen-q4-vanilla17b,chat,8,58,0.2197,0.1640,0.1788,0.2661
30
+ qwen-q4-vanilla17b,chat,9,58,0.1945,0.1380,0.1544,0.2391
31
+ qwen-q4-vanilla17b,chat,10,58,0.1752,0.1145,0.1350,0.2199
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-vanilla17b,math,1,60,0.8614,0.8695,0.8469,0.8759
3
+ qwen-q4-vanilla17b,math,2,60,0.7467,0.7370,0.7221,0.7711
4
+ qwen-q4-vanilla17b,math,3,60,0.6642,0.6555,0.6354,0.6929
5
+ qwen-q4-vanilla17b,math,4,60,0.5917,0.5945,0.5596,0.6233
6
+ qwen-q4-vanilla17b,math,5,60,0.5365,0.5255,0.5022,0.5705
7
+ qwen-q4-vanilla17b,code,1,60,0.8306,0.8225,0.8161,0.8458
8
+ qwen-q4-vanilla17b,code,2,60,0.6994,0.6960,0.6791,0.7209
9
+ qwen-q4-vanilla17b,code,3,60,0.5953,0.5850,0.5719,0.6201
10
+ qwen-q4-vanilla17b,code,4,60,0.5117,0.4920,0.4882,0.5382
11
+ qwen-q4-vanilla17b,code,5,60,0.4440,0.4175,0.4158,0.4744
12
+ qwen-q4-vanilla17b,chat,1,58,0.7753,0.7690,0.7534,0.7985
13
+ qwen-q4-vanilla17b,chat,2,58,0.6074,0.5805,0.5755,0.6407
14
+ qwen-q4-vanilla17b,chat,3,58,0.4889,0.4695,0.4515,0.5299
15
+ qwen-q4-vanilla17b,chat,4,58,0.4014,0.3630,0.3599,0.4463
16
+ qwen-q4-vanilla17b,chat,5,58,0.3373,0.2830,0.2937,0.3847
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q4-vanilla17b,math,1,60,0.8568,0.8570,0.8415,0.8715
3
+ qwen-q4-vanilla17b,math,2,60,0.7372,0.7320,0.7132,0.7605
4
+ qwen-q4-vanilla17b,math,3,60,0.6551,0.6560,0.6267,0.6840
5
+ qwen-q4-vanilla17b,math,4,60,0.5800,0.5900,0.5486,0.6121
6
+ qwen-q4-vanilla17b,math,5,60,0.5246,0.5210,0.4935,0.5572
7
+ qwen-q4-vanilla17b,math,6,60,0.4753,0.4745,0.4427,0.5094
8
+ qwen-q4-vanilla17b,math,7,60,0.4283,0.4170,0.3954,0.4638
9
+ qwen-q4-vanilla17b,code,1,60,0.8222,0.8260,0.8074,0.8375
10
+ qwen-q4-vanilla17b,code,2,60,0.6858,0.6730,0.6650,0.7075
11
+ qwen-q4-vanilla17b,code,3,60,0.5783,0.5640,0.5553,0.6035
12
+ qwen-q4-vanilla17b,code,4,60,0.4988,0.4785,0.4743,0.5257
13
+ qwen-q4-vanilla17b,code,5,60,0.4323,0.4150,0.4042,0.4638
14
+ qwen-q4-vanilla17b,code,6,60,0.3756,0.3580,0.3435,0.4094
15
+ qwen-q4-vanilla17b,code,7,60,0.3322,0.3050,0.3009,0.3676
16
+ qwen-q4-vanilla17b,chat,1,58,0.7715,0.7615,0.7502,0.7939
17
+ qwen-q4-vanilla17b,chat,2,58,0.5965,0.5805,0.5665,0.6277
18
+ qwen-q4-vanilla17b,chat,3,58,0.4779,0.4650,0.4422,0.5168
19
+ qwen-q4-vanilla17b,chat,4,58,0.3930,0.3585,0.3511,0.4369
20
+ qwen-q4-vanilla17b,chat,5,58,0.3258,0.2680,0.2832,0.3719
21
+ qwen-q4-vanilla17b,chat,6,58,0.2849,0.2140,0.2400,0.3335
22
+ qwen-q4-vanilla17b,chat,7,58,0.2500,0.1795,0.2042,0.3002
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q8-eagle3,math,1,60,0.6384,0.6310,0.6204,0.6561
3
+ qwen-q8-eagle3,math,2,60,0.4114,0.4060,0.3919,0.4320
4
+ qwen-q8-eagle3,math,3,60,0.2123,0.2140,0.1969,0.2300
5
+ qwen-q8-eagle3,math,4,60,0.1168,0.1140,0.1051,0.1296
6
+ qwen-q8-eagle3,math,5,60,0.0649,0.0620,0.0564,0.0746
7
+ qwen-q8-eagle3,math,6,60,0.0355,0.0315,0.0295,0.0418
8
+ qwen-q8-eagle3,math,7,60,0.0199,0.0190,0.0161,0.0240
9
+ qwen-q8-eagle3,math,8,60,0.0110,0.0100,0.0084,0.0138
10
+ qwen-q8-eagle3,math,9,60,0.0068,0.0040,0.0049,0.0088
11
+ qwen-q8-eagle3,math,10,60,0.0043,0.0000,0.0027,0.0059
12
+ qwen-q8-eagle3,code,1,60,0.6466,0.6500,0.6291,0.6649
13
+ qwen-q8-eagle3,code,2,60,0.4087,0.4010,0.3883,0.4312
14
+ qwen-q8-eagle3,code,3,60,0.1994,0.1765,0.1784,0.2247
15
+ qwen-q8-eagle3,code,4,60,0.1104,0.0920,0.0901,0.1367
16
+ qwen-q8-eagle3,code,5,60,0.0654,0.0450,0.0475,0.0899
17
+ qwen-q8-eagle3,code,6,60,0.0408,0.0200,0.0257,0.0614
18
+ qwen-q8-eagle3,code,7,60,0.0283,0.0100,0.0170,0.0426
19
+ qwen-q8-eagle3,code,8,60,0.0205,0.0080,0.0114,0.0317
20
+ qwen-q8-eagle3,code,9,60,0.0127,0.0000,0.0064,0.0214
21
+ qwen-q8-eagle3,code,10,60,0.0092,0.0000,0.0035,0.0173
22
+ qwen-q8-eagle3,chat,1,58,0.5779,0.6090,0.5295,0.6233
23
+ qwen-q8-eagle3,chat,2,58,0.3406,0.3450,0.2972,0.3811
24
+ qwen-q8-eagle3,chat,3,58,0.1728,0.1625,0.1398,0.2089
25
+ qwen-q8-eagle3,chat,4,58,0.1127,0.0845,0.0830,0.1480
26
+ qwen-q8-eagle3,chat,5,58,0.0690,0.0455,0.0468,0.0959
27
+ qwen-q8-eagle3,chat,6,58,0.0538,0.0280,0.0343,0.0776
28
+ qwen-q8-eagle3,chat,7,58,0.0381,0.0095,0.0232,0.0562
29
+ qwen-q8-eagle3,chat,8,58,0.0309,0.0090,0.0170,0.0477
30
+ qwen-q8-eagle3,chat,9,58,0.0237,0.0000,0.0121,0.0382
31
+ qwen-q8-eagle3,chat,10,58,0.0181,0.0000,0.0079,0.0312
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q8-eagle3,math,1,60,0.6440,0.6290,0.6264,0.6617
3
+ qwen-q8-eagle3,math,2,60,0.4159,0.4220,0.3966,0.4358
4
+ qwen-q8-eagle3,math,3,60,0.2202,0.2165,0.2050,0.2356
5
+ qwen-q8-eagle3,math,4,60,0.1221,0.1155,0.1107,0.1348
6
+ qwen-q8-eagle3,math,5,60,0.0690,0.0630,0.0602,0.0790
7
+ qwen-q8-eagle3,code,1,60,0.6543,0.6465,0.6373,0.6724
8
+ qwen-q8-eagle3,code,2,60,0.4166,0.4120,0.3953,0.4391
9
+ qwen-q8-eagle3,code,3,60,0.2039,0.1845,0.1819,0.2286
10
+ qwen-q8-eagle3,code,4,60,0.1145,0.0900,0.0935,0.1394
11
+ qwen-q8-eagle3,code,5,60,0.0663,0.0450,0.0489,0.0882
12
+ qwen-q8-eagle3,chat,1,58,0.5884,0.6265,0.5389,0.6331
13
+ qwen-q8-eagle3,chat,2,58,0.3518,0.3700,0.3078,0.3941
14
+ qwen-q8-eagle3,chat,3,58,0.1824,0.1740,0.1497,0.2200
15
+ qwen-q8-eagle3,chat,4,58,0.1178,0.0885,0.0903,0.1517
16
+ qwen-q8-eagle3,chat,5,58,0.0723,0.0445,0.0506,0.0976
experiments/analysis/curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ config,domain,position,n,alpha_mean,alpha_median,ci95_low,ci95_high
2
+ qwen-q8-eagle3,math,1,60,0.6422,0.6315,0.6244,0.6602
3
+ qwen-q8-eagle3,math,2,60,0.4124,0.4095,0.3926,0.4324
4
+ qwen-q8-eagle3,math,3,60,0.2172,0.2140,0.2027,0.2328
5
+ qwen-q8-eagle3,math,4,60,0.1193,0.1155,0.1080,0.1315
6
+ qwen-q8-eagle3,math,5,60,0.0672,0.0630,0.0587,0.0769
7
+ qwen-q8-eagle3,math,6,60,0.0396,0.0355,0.0337,0.0458
8
+ qwen-q8-eagle3,math,7,60,0.0225,0.0195,0.0182,0.0270
9
+ qwen-q8-eagle3,code,1,60,0.6502,0.6410,0.6335,0.6674
10
+ qwen-q8-eagle3,code,2,60,0.4082,0.4040,0.3865,0.4310
11
+ qwen-q8-eagle3,code,3,60,0.1971,0.1840,0.1773,0.2205
12
+ qwen-q8-eagle3,code,4,60,0.1089,0.0880,0.0905,0.1322
13
+ qwen-q8-eagle3,code,5,60,0.0638,0.0430,0.0474,0.0862
14
+ qwen-q8-eagle3,code,6,60,0.0379,0.0200,0.0249,0.0543
15
+ qwen-q8-eagle3,code,7,60,0.0276,0.0090,0.0166,0.0418
16
+ qwen-q8-eagle3,chat,1,58,0.5867,0.6200,0.5382,0.6317
17
+ qwen-q8-eagle3,chat,2,58,0.3437,0.3615,0.3013,0.3834
18
+ qwen-q8-eagle3,chat,3,58,0.1738,0.1685,0.1435,0.2065
19
+ qwen-q8-eagle3,chat,4,58,0.1146,0.0855,0.0876,0.1466
20
+ qwen-q8-eagle3,chat,5,58,0.0699,0.0415,0.0478,0.0961
21
+ qwen-q8-eagle3,chat,6,58,0.0515,0.0190,0.0318,0.0751
22
+ qwen-q8-eagle3,chat,7,58,0.0363,0.0095,0.0214,0.0545