diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..1971984b781c712f3a536ccfcabde80c7d71bc1c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fa8cdd645c9394eee2b0e0745129094ca6ec6bb1725ebe2f58a661f7c2c4155 +size 39216 diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.json b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.json new file mode 100644 index 0000000000000000000000000000000000000000..bc5ca79fbe7f097dd741d2b7326f0add61904275 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.json @@ -0,0 +1,142 @@ +{ + "cost_model": { + "currency": "gpu_s", + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "ref": { + "tier": "think16k", + "acc": 0.9279941219691403, + "equiv": 5397.500408759384 + }, + "frontier": [ + { + "R": 2000, + "acc": 0.7883908890521675, + "equiv": 1129.5711941564073 + }, + { + "R": 4000, + "acc": 0.7883908890521675, + "equiv": 1129.5711941564073 + }, + { + "R": 6000, + "acc": 0.7883908890521675, + "equiv": 1129.5711941564073 + }, + { + "R": 8000, + "acc": 0.7883908890521675, + "equiv": 1129.5711941564073 + }, + { + "R": 12000, + "acc": 0.7883908890521675, + "equiv": 1129.5711941564073 + }, + { + "R": 16000, + "acc": 0.837304502991498, + "equiv": 2834.403313721442 + }, + { + "R": 24000, + "acc": 0.8872152828802351, + "equiv": 4230.600976175624 + }, + { + "R": 32000, + "acc": 0.9125118085441378, + "equiv": 4900.948252343457 + }, + { + "R": 48000, + "acc": 0.9316678912564291, + "equiv": 5448.530278926885 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9294636296840558, + "equiv": 5450.9879613091925, + "R": [ + 44381.251420165536, + 49612.630326552375 + ], + "delta_vs_agree2": [ + 0.0014695077149155045, + -0.005405689094153458, + 0.008187257268814946 + ], + "actions": { + "mid:think16k": 369 + } + }, + { + "seed": 1, + "acc": 0.9267345439277841, + "equiv": 5334.087495098382, + "R": [ + 50191.39018487453, + 46282.92419459038 + ], + "delta_vs_agree2": [ + -0.0012595780413561446, + -0.008239739687204786, + 0.0058268605017319135 + ], + "actions": { + "mid:think16k": 335 + } + }, + { + "seed": 2, + "acc": 0.930985619817361, + "equiv": 5409.716893546653, + "R": [ + 50976.04938951666, + 46873.10056327461 + ], + "delta_vs_agree2": [ + 0.002991497848220847, + -0.003673769287288757, + 0.009761729820510129 + ], + "actions": { + "mid:think16k": 349 + } + }, + { + "seed": 3, + "acc": 0.9316154088380394, + "equiv": 5388.791856758997, + "R": [ + 43948.42366826647, + 47852.21935253754 + ], + "delta_vs_agree2": [ + 0.0036212868688989203, + -0.0027828802351212334, + 0.010129106749239005 + ], + "actions": { + "mid:think16k": 358 + } + } + ], + "agree2_by_tier": { + "think16k": { + "acc": 0.9279941219691403, + "equiv": 5397.500408759384, + "delta_vs_ref": [ + 0.0, + 0.0, + 0.0 + ] + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.log b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.log new file mode 100644 index 0000000000000000000000000000000000000000..a9b6f29173e480d47ebe33280b70571e018e72aa --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.log @@ -0,0 +1,17 @@ +2026-10-25 14:12:14,778 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 -> think16k: acc 0.928 equiv 5398 | base 0.788 at 1130 | always think16k 0.948 at 8284 +cost model gpu_s: 1 equiv = 0.0006341 GPU-seconds +agree2 -> think16k acc 0.928 equiv 5398 ( +0) | vs reference +0.0 [+0.0, +0.0] +R 2000: acc 0.788 equiv 1130 +R 4000: acc 0.788 equiv 1130 +R 6000: acc 0.788 equiv 1130 +R 8000: acc 0.788 equiv 1130 +R 12000: acc 0.788 equiv 1130 +R 16000: acc 0.837 equiv 2834 +R 24000: acc 0.887 equiv 4231 +R 32000: acc 0.913 equiv 4901 +R 48000: acc 0.932 equiv 5449 +parity: Bayes at agree2's cost acc 0.929 equiv 5451 (+53) | vs agree2 +0.1 [-0.5, +0.8] | R [44381, 49613] | actions {'mid:think16k': 369} +split 1: Bayes at agree2's cost acc 0.927 equiv 5334 (-63) | vs agree2 -0.1 [-0.8, +0.6] | R [50191, 46283] | actions {'mid:think16k': 335} +split 2: Bayes at agree2's cost acc 0.931 equiv 5410 (+12) | vs agree2 +0.3 [-0.4, +1.0] | R [50976, 46873] | actions {'mid:think16k': 349} +split 3: Bayes at agree2's cost acc 0.932 equiv 5389 (-9) | vs agree2 +0.4 [-0.3, +1.0] | R [43948, 47852] | actions {'mid:think16k': 358} diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.provenance.json b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f91b5841a4f384917d71c4a29989fb93bb163dd0 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_gpu_s.provenance.json @@ -0,0 +1,98 @@ +{ + "started_at": "2026-10-25T14:12:12+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_control.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--prm", + "workdir/selector/pool_mth_k8_chat/prm_selector.json", + "--prm-reads", + "workdir/selector/pool_mth_k8_chat/prm_reads.json", + "--step-tokens", + "", + "--critics", + "agree", + "--task-prior", + "workdir/bayes/mth_task_prior.json", + "--tbins", + "3", + "--cost-model", + "gpu_s", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--out", + "workdir/bayes/v8/bc8_gpu_s.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "prm": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/prm_selector.json", + "bytes": 9360982, + "sha256": "0be50791aa0d0af5d2f2d239f8fdf551919f716c2c84b295553de06be28a3fcd" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:12:31+04:00", + "wall_secs": 18.9 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..7eacec08883eef54e9eedea251d7dc3ec89c945e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e8f9347986681a7d5bc15a13c5e1e8522cd048e20aea56da1b7f8c4da4c2af +size 22883 diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.json b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..9d72a655fec4f41edf907dc90cfa19d66bfa404e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.json @@ -0,0 +1,142 @@ +{ + "cost_model": { + "currency": "tokens", + "unit": "base tokens", + "per_equiv": 1.0, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "ref": { + "tier": "think16k", + "acc": 0.9279941219691403, + "equiv": 4617.346383961372 + }, + "frontier": [ + { + "R": 2000, + "acc": 0.7883908890521675, + "equiv": 1104.853783982366 + }, + { + "R": 4000, + "acc": 0.7883908890521675, + "equiv": 1104.853783982366 + }, + { + "R": 6000, + "acc": 0.7883908890521675, + "equiv": 1104.853783982366 + }, + { + "R": 8000, + "acc": 0.7883908890521675, + "equiv": 1104.853783982366 + }, + { + "R": 12000, + "acc": 0.8320037787341241, + "equiv": 2256.260165844442 + }, + { + "R": 16000, + "acc": 0.8729400650781987, + "equiv": 3147.0462894930197 + }, + { + "R": 24000, + "acc": 0.9096777579510863, + "equiv": 3944.4870998215597 + }, + { + "R": 32000, + "acc": 0.9225884328749868, + "equiv": 4257.002844547077 + }, + { + "R": 48000, + "acc": 0.9362338616563451, + "equiv": 4676.8411986984365 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9361288968195656, + "equiv": 4655.3114411672095, + "R": [ + 48361.3395565794, + 44732.65597091439 + ], + "delta_vs_agree2": [ + 0.008134774850425108, + 0.0022554319303033487, + 0.014592736433294834 + ], + "actions": { + "mid:think16k": 466 + } + }, + { + "seed": 1, + "acc": 0.9306182428886324, + "equiv": 4628.1536895140125, + "R": [ + 48097.040311758275, + 50399.93104936121 + ], + "delta_vs_agree2": [ + 0.002624120919491971, + -0.004043770336937124, + 0.009711871523039768 + ], + "actions": { + "mid:think16k": 451 + } + }, + { + "seed": 2, + "acc": 0.9324551275322766, + "equiv": 4598.262076204472, + "R": [ + 49825.98952869387, + 47632.1174822072 + ], + "delta_vs_agree2": [ + 0.004461005563136351, + -0.0022567439907630934, + 0.011283719953815472 + ], + "actions": { + "mid:think16k": 439 + } + }, + { + "seed": 3, + "acc": 0.9348168363598194, + "equiv": 4613.861110527973, + "R": [ + 45769.134996550725, + 51481.074300937435 + ], + "delta_vs_agree2": [ + 0.006822714390679124, + 0.0006284769602183275, + 0.013120604597459852 + ], + "actions": { + "mid:think16k": 441 + } + } + ], + "agree2_by_tier": { + "think16k": { + "acc": 0.9279941219691403, + "equiv": 4617.346383961372, + "delta_vs_ref": [ + 0.0, + 0.0, + 0.0 + ] + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.log b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.log new file mode 100644 index 0000000000000000000000000000000000000000..5620298f8f5beb575331306ea8c36c00c5e0cbb7 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.log @@ -0,0 +1,17 @@ +2026-10-25 14:12:14,988 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 -> think16k: acc 0.928 equiv 4617 | base 0.788 at 1105 | always think16k 0.948 at 6341 +cost model tokens: 1 equiv = 1 base tokens +agree2 -> think16k acc 0.928 equiv 4617 ( +0) | vs reference +0.0 [+0.0, +0.0] +R 2000: acc 0.788 equiv 1105 +R 4000: acc 0.788 equiv 1105 +R 6000: acc 0.788 equiv 1105 +R 8000: acc 0.788 equiv 1105 +R 12000: acc 0.832 equiv 2256 +R 16000: acc 0.873 equiv 3147 +R 24000: acc 0.910 equiv 3944 +R 32000: acc 0.923 equiv 4257 +R 48000: acc 0.936 equiv 4677 +parity: Bayes at agree2's cost acc 0.936 equiv 4655 (+38) | vs agree2 +0.8 [+0.2, +1.5] | R [48361, 44733] | actions {'mid:think16k': 466} +split 1: Bayes at agree2's cost acc 0.931 equiv 4628 (+11) | vs agree2 +0.3 [-0.4, +1.0] | R [48097, 50400] | actions {'mid:think16k': 451} +split 2: Bayes at agree2's cost acc 0.932 equiv 4598 (-19) | vs agree2 +0.4 [-0.2, +1.1] | R [49826, 47632] | actions {'mid:think16k': 439} +split 3: Bayes at agree2's cost acc 0.935 equiv 4614 (-3) | vs agree2 +0.7 [+0.1, +1.3] | R [45769, 51481] | actions {'mid:think16k': 441} diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.provenance.json b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0a7fa9d8da3604b008de67da982b700a4141495e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_tokens.provenance.json @@ -0,0 +1,98 @@ +{ + "started_at": "2026-10-25T14:12:12+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_control.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--prm", + "workdir/selector/pool_mth_k8_chat/prm_selector.json", + "--prm-reads", + "workdir/selector/pool_mth_k8_chat/prm_reads.json", + "--step-tokens", + "", + "--critics", + "agree", + "--task-prior", + "workdir/bayes/mth_task_prior.json", + "--tbins", + "3", + "--cost-model", + "tokens", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json", + "--out", + "workdir/bayes/v8/bc8_tokens.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "prm": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/prm_selector.json", + "bytes": 9360982, + "sha256": "0be50791aa0d0af5d2f2d239f8fdf551919f716c2c84b295553de06be28a3fcd" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:12:31+04:00", + "wall_secs": 18.6 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..d07a8c8ffdf526616adf50eaaa63967bbaa4fec0 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d00641ff88ba44660e952983378e9b60bdbe41fa72386e90da50d770c08d73 +size 21885 diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.json new file mode 100644 index 0000000000000000000000000000000000000000..78ee6e1c84670a7199c8810c10e5866a4aded947 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.json @@ -0,0 +1,142 @@ +{ + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "cheapest", + "base": "qwen3-8b:nothink" + }, + "ref": { + "tier": "think16k", + "acc": 0.9279941219691403, + "equiv": 4668.220714810538 + }, + "frontier": [ + { + "R": 2000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 4000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 6000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 8000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 12000, + "acc": 0.831373989713446, + "equiv": 2256.5245617718065 + }, + { + "R": 16000, + "acc": 0.8714705573632832, + "equiv": 3156.445024666737 + }, + { + "R": 24000, + "acc": 0.9091529337671879, + "equiv": 3961.4649679857243 + }, + { + "R": 32000, + "acc": 0.9218536790175291, + "equiv": 4277.911934501942 + }, + { + "R": 48000, + "acc": 0.9362338616563451, + "equiv": 4706.574734963788 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9362338616563451, + "equiv": 4706.6886742941115, + "R": [ + 48781.018757585625, + 47737.196817866185 + ], + "delta_vs_agree2": [ + 0.008239739687204787, + 0.0023617088275427736, + 0.014697701270074514 + ], + "actions": { + "mid:think16k": 475 + } + }, + { + "seed": 1, + "acc": 0.9318778209299884, + "equiv": 4678.954576466884, + "R": [ + 50114.318176677625, + 50034.462920524595 + ], + "delta_vs_agree2": [ + 0.0038836989608481164, + -0.0025204681431720364, + 0.010970137503936177 + ], + "actions": { + "mid:think16k": 463 + } + }, + { + "seed": 2, + "acc": 0.9338196704104126, + "equiv": 4655.15776214968, + "R": [ + 49992.40866974007, + 50651.911602932625 + ], + "delta_vs_agree2": [ + 0.005825548441272175, + -0.000526136244358139, + 0.012229715545292322 + ], + "actions": { + "mid:think16k": 443 + } + }, + { + "seed": 3, + "acc": 0.9354466253804977, + "equiv": 4696.135798257584, + "R": [ + 48932.595912206365, + 51974.5228064271 + ], + "delta_vs_agree2": [ + 0.007452503411357195, + 0.0012595780413561463, + 0.013699223260207826 + ], + "actions": { + "mid:think16k": 463 + } + } + ], + "agree2_by_tier": { + "think16k": { + "acc": 0.9279941219691403, + "equiv": 4668.220714810538, + "delta_vs_ref": [ + 0.0, + 0.0, + 0.0 + ] + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.log b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.log new file mode 100644 index 0000000000000000000000000000000000000000..b2ece448496f3015a4228830d44eb8abd05fccb0 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.log @@ -0,0 +1,17 @@ +2026-10-25 14:12:14,966 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 -> think16k: acc 0.928 equiv 4668 | base 0.788 at 1130 | always think16k 0.948 at 6346 +cost model usd: 1 equiv = 2e-07 USD +agree2 -> think16k acc 0.928 equiv 4668 ( +0) | vs reference +0.0 [+0.0, +0.0] +R 2000: acc 0.788 equiv 1130 +R 4000: acc 0.788 equiv 1130 +R 6000: acc 0.788 equiv 1130 +R 8000: acc 0.788 equiv 1130 +R 12000: acc 0.831 equiv 2257 +R 16000: acc 0.871 equiv 3156 +R 24000: acc 0.909 equiv 3961 +R 32000: acc 0.922 equiv 4278 +R 48000: acc 0.936 equiv 4707 +parity: Bayes at agree2's cost acc 0.936 equiv 4707 (+38) | vs agree2 +0.8 [+0.2, +1.5] | R [48781, 47737] | actions {'mid:think16k': 475} +split 1: Bayes at agree2's cost acc 0.932 equiv 4679 (+11) | vs agree2 +0.4 [-0.3, +1.1] | R [50114, 50034] | actions {'mid:think16k': 463} +split 2: Bayes at agree2's cost acc 0.934 equiv 4655 (-13) | vs agree2 +0.6 [-0.1, +1.2] | R [49992, 50652] | actions {'mid:think16k': 443} +split 3: Bayes at agree2's cost acc 0.935 equiv 4696 (+28) | vs agree2 +0.7 [+0.1, +1.4] | R [48933, 51975] | actions {'mid:think16k': 463} diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.provenance.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7c4cece311b4fe8e312a34609939208fc8c6e241 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_cheapest.provenance.json @@ -0,0 +1,100 @@ +{ + "started_at": "2026-10-25T14:12:12+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_control.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--prm", + "workdir/selector/pool_mth_k8_chat/prm_selector.json", + "--prm-reads", + "workdir/selector/pool_mth_k8_chat/prm_reads.json", + "--step-tokens", + "", + "--critics", + "agree", + "--task-prior", + "workdir/bayes/mth_task_prior.json", + "--tbins", + "3", + "--cost-model", + "usd", + "--price-scenario", + "cheapest", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--out", + "workdir/bayes/v8/bc8_usd_cheapest.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "prm": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/prm_selector.json", + "bytes": 9360982, + "sha256": "0be50791aa0d0af5d2f2d239f8fdf551919f716c2c84b295553de06be28a3fcd" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:12:31+04:00", + "wall_secs": 18.9 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..d07a8c8ffdf526616adf50eaaa63967bbaa4fec0 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d00641ff88ba44660e952983378e9b60bdbe41fa72386e90da50d770c08d73 +size 21885 diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.json new file mode 100644 index 0000000000000000000000000000000000000000..36b8f4583285528edbc5485feaface4ab2149431 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.json @@ -0,0 +1,142 @@ +{ + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "dear_32b", + "base": "qwen3-8b:nothink" + }, + "ref": { + "tier": "think16k", + "acc": 0.9279941219691403, + "equiv": 4668.220714810538 + }, + "frontier": [ + { + "R": 2000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 4000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 6000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 8000, + "acc": 0.7883908890521675, + "equiv": 1129.5194709772227 + }, + { + "R": 12000, + "acc": 0.831373989713446, + "equiv": 2256.5245617718065 + }, + { + "R": 16000, + "acc": 0.8714705573632832, + "equiv": 3156.445024666737 + }, + { + "R": 24000, + "acc": 0.9091529337671879, + "equiv": 3961.4649679857243 + }, + { + "R": 32000, + "acc": 0.9218536790175291, + "equiv": 4277.911934501942 + }, + { + "R": 48000, + "acc": 0.9362338616563451, + "equiv": 4706.574734963788 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9362338616563451, + "equiv": 4706.6886742941115, + "R": [ + 48781.018757585625, + 47737.196817866185 + ], + "delta_vs_agree2": [ + 0.008239739687204787, + 0.0023617088275427736, + 0.014697701270074514 + ], + "actions": { + "mid:think16k": 475 + } + }, + { + "seed": 1, + "acc": 0.9318778209299884, + "equiv": 4678.954576466884, + "R": [ + 50114.318176677625, + 50034.462920524595 + ], + "delta_vs_agree2": [ + 0.0038836989608481164, + -0.0025204681431720364, + 0.010970137503936177 + ], + "actions": { + "mid:think16k": 463 + } + }, + { + "seed": 2, + "acc": 0.9338196704104126, + "equiv": 4655.15776214968, + "R": [ + 49992.40866974007, + 50651.911602932625 + ], + "delta_vs_agree2": [ + 0.005825548441272175, + -0.000526136244358139, + 0.012229715545292322 + ], + "actions": { + "mid:think16k": 443 + } + }, + { + "seed": 3, + "acc": 0.9354466253804977, + "equiv": 4696.135798257584, + "R": [ + 48932.595912206365, + 51974.5228064271 + ], + "delta_vs_agree2": [ + 0.007452503411357195, + 0.0012595780413561463, + 0.013699223260207826 + ], + "actions": { + "mid:think16k": 463 + } + } + ], + "agree2_by_tier": { + "think16k": { + "acc": 0.9279941219691403, + "equiv": 4668.220714810538, + "delta_vs_ref": [ + 0.0, + 0.0, + 0.0 + ] + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.log b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.log new file mode 100644 index 0000000000000000000000000000000000000000..d978999a84b6bae0482fbedbc1417b326ff9ac00 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.log @@ -0,0 +1,17 @@ +2026-10-25 14:12:15,010 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 -> think16k: acc 0.928 equiv 4668 | base 0.788 at 1130 | always think16k 0.948 at 6346 +cost model usd: 1 equiv = 2e-07 USD +agree2 -> think16k acc 0.928 equiv 4668 ( +0) | vs reference +0.0 [+0.0, +0.0] +R 2000: acc 0.788 equiv 1130 +R 4000: acc 0.788 equiv 1130 +R 6000: acc 0.788 equiv 1130 +R 8000: acc 0.788 equiv 1130 +R 12000: acc 0.831 equiv 2257 +R 16000: acc 0.871 equiv 3156 +R 24000: acc 0.909 equiv 3961 +R 32000: acc 0.922 equiv 4278 +R 48000: acc 0.936 equiv 4707 +parity: Bayes at agree2's cost acc 0.936 equiv 4707 (+38) | vs agree2 +0.8 [+0.2, +1.5] | R [48781, 47737] | actions {'mid:think16k': 475} +split 1: Bayes at agree2's cost acc 0.932 equiv 4679 (+11) | vs agree2 +0.4 [-0.3, +1.1] | R [50114, 50034] | actions {'mid:think16k': 463} +split 2: Bayes at agree2's cost acc 0.934 equiv 4655 (-13) | vs agree2 +0.6 [-0.1, +1.2] | R [49992, 50652] | actions {'mid:think16k': 443} +split 3: Bayes at agree2's cost acc 0.935 equiv 4696 (+28) | vs agree2 +0.7 [+0.1, +1.4] | R [48933, 51975] | actions {'mid:think16k': 463} diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.provenance.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..362a1162c55b49d8ab39371ecd443f7ace31deaf --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_dear_32b.provenance.json @@ -0,0 +1,100 @@ +{ + "started_at": "2026-10-25T14:12:12+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_control.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--prm", + "workdir/selector/pool_mth_k8_chat/prm_selector.json", + "--prm-reads", + "workdir/selector/pool_mth_k8_chat/prm_reads.json", + "--step-tokens", + "", + "--critics", + "agree", + "--task-prior", + "workdir/bayes/mth_task_prior.json", + "--tbins", + "3", + "--cost-model", + "usd", + "--price-scenario", + "dear_32b", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--out", + "workdir/bayes/v8/bc8_usd_dear_32b.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "prm": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/prm_selector.json", + "bytes": 9360982, + "sha256": "0be50791aa0d0af5d2f2d239f8fdf551919f716c2c84b295553de06be28a3fcd" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:12:31+04:00", + "wall_secs": 18.6 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..b7924517f138c04e4b158435270384c9fda6774c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35f44d70ba65998832380bcec974a2b88e323c0b5bee86dc0fec50b0a4a630e0 +size 27376 diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.json new file mode 100644 index 0000000000000000000000000000000000000000..334a502d92f2d2695c29096d01a8cc4a3de9ad91 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.json @@ -0,0 +1,142 @@ +{ + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "scenario": "openrouter_only", + "base": "qwen3-8b:nothink" + }, + "ref": { + "tier": "think16k", + "acc": 0.9279941219691403, + "equiv": 4669.85060504731 + }, + "frontier": [ + { + "R": 2000, + "acc": 0.7883908890521675, + "equiv": 1130.2242048913613 + }, + { + "R": 4000, + "acc": 0.7883908890521675, + "equiv": 1130.2242048913613 + }, + { + "R": 6000, + "acc": 0.7883908890521675, + "equiv": 1130.2242048913613 + }, + { + "R": 8000, + "acc": 0.7883908890521675, + "equiv": 1130.2242048913613 + }, + { + "R": 12000, + "acc": 0.831373989713446, + "equiv": 2257.3525633912636 + }, + { + "R": 16000, + "acc": 0.8714705573632832, + "equiv": 3157.4179504865865 + }, + { + "R": 24000, + "acc": 0.9091529337671879, + "equiv": 3962.602664607356 + }, + { + "R": 32000, + "acc": 0.9218536790175291, + "equiv": 4279.105650107214 + }, + { + "R": 48000, + "acc": 0.9362338616563451, + "equiv": 4707.833017439158 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9362338616563451, + "equiv": 4704.38500952181, + "R": [ + 48776.77570445791, + 47756.578806575955 + ], + "delta_vs_agree2": [ + 0.008239739687204787, + 0.0023617088275427736, + 0.014697701270074514 + ], + "actions": { + "mid:think16k": 475 + } + }, + { + "seed": 1, + "acc": 0.9318778209299884, + "equiv": 4689.905929013781, + "R": [ + 50112.86512598025, + 50031.56149195534 + ], + "delta_vs_agree2": [ + 0.0038836989608481164, + -0.0025204681431720364, + 0.010970137503936177 + ], + "actions": { + "mid:think16k": 467 + } + }, + { + "seed": 2, + "acc": 0.9338196704104126, + "equiv": 4656.432901977837, + "R": [ + 49992.40866974007, + 50648.97436937731 + ], + "delta_vs_agree2": [ + 0.005825548441272175, + -0.000526136244358139, + 0.012229715545292322 + ], + "actions": { + "mid:think16k": 443 + } + }, + { + "seed": 3, + "acc": 0.9354466253804977, + "equiv": 4684.456896939525, + "R": [ + 48931.177125216906, + 51997.132852835726 + ], + "delta_vs_agree2": [ + 0.007452503411357195, + 0.0012595780413561463, + 0.013699223260207826 + ], + "actions": { + "mid:think16k": 460 + } + } + ], + "agree2_by_tier": { + "think16k": { + "acc": 0.9279941219691403, + "equiv": 4669.85060504731, + "delta_vs_ref": [ + 0.0, + 0.0, + 0.0 + ] + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.log b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.log new file mode 100644 index 0000000000000000000000000000000000000000..32e2060ee44cca96d616a3f6fac0be405581b35c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.log @@ -0,0 +1,17 @@ +2026-10-25 14:12:14,989 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 -> think16k: acc 0.928 equiv 4670 | base 0.788 at 1130 | always think16k 0.948 at 6347 +cost model usd: 1 equiv = 4.55e-07 USD +agree2 -> think16k acc 0.928 equiv 4670 ( +0) | vs reference +0.0 [+0.0, +0.0] +R 2000: acc 0.788 equiv 1130 +R 4000: acc 0.788 equiv 1130 +R 6000: acc 0.788 equiv 1130 +R 8000: acc 0.788 equiv 1130 +R 12000: acc 0.831 equiv 2257 +R 16000: acc 0.871 equiv 3157 +R 24000: acc 0.909 equiv 3963 +R 32000: acc 0.922 equiv 4279 +R 48000: acc 0.936 equiv 4708 +parity: Bayes at agree2's cost acc 0.936 equiv 4704 (+35) | vs agree2 +0.8 [+0.2, +1.5] | R [48777, 47757] | actions {'mid:think16k': 475} +split 1: Bayes at agree2's cost acc 0.932 equiv 4690 (+20) | vs agree2 +0.4 [-0.3, +1.1] | R [50113, 50032] | actions {'mid:think16k': 467} +split 2: Bayes at agree2's cost acc 0.934 equiv 4656 (-13) | vs agree2 +0.6 [-0.1, +1.2] | R [49992, 50649] | actions {'mid:think16k': 443} +split 3: Bayes at agree2's cost acc 0.935 equiv 4684 (+15) | vs agree2 +0.7 [+0.1, +1.4] | R [48931, 51997] | actions {'mid:think16k': 460} diff --git a/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.provenance.json b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..1b655382a6815e1349a62da5af427676182003f8 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/bc8_usd_openrouter_only.provenance.json @@ -0,0 +1,100 @@ +{ + "started_at": "2026-10-25T14:12:12+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_control.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--prm", + "workdir/selector/pool_mth_k8_chat/prm_selector.json", + "--prm-reads", + "workdir/selector/pool_mth_k8_chat/prm_reads.json", + "--step-tokens", + "", + "--critics", + "agree", + "--task-prior", + "workdir/bayes/mth_task_prior.json", + "--tbins", + "3", + "--cost-model", + "usd", + "--price-scenario", + "openrouter_only", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--out", + "workdir/bayes/v8/bc8_usd_openrouter_only.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "prm": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/prm_selector.json", + "bytes": 9360982, + "sha256": "0be50791aa0d0af5d2f2d239f8fdf551919f716c2c84b295553de06be28a3fcd" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:12:31+04:00", + "wall_secs": 18.4 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/currency_table.json b/analysis/bayes_sc_2026-10-25/v8/currency_table.json new file mode 100644 index 0000000000000000000000000000000000000000..6dea10d82b32d79b7a421b83b2f1042788ad2a68 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/currency_table.json @@ -0,0 +1,1287 @@ +{ + "tokens": { + "unit": "base tokens", + "per_equiv": 1.0, + "hull": [ + { + "cost": 1124.9408523144746, + "acc": 0.7907788390889052, + "policy": "scboth:SC@1" + }, + { + "cost": 4283.8179463629685, + "acc": 0.9405767817781043, + "policy": "sc32:agree2" + }, + { + "cost": 5317.299044819985, + "acc": 0.9603232916972814, + "policy": "sc32:always think" + } + ], + "controllers": { + "bc8": { + "acc_minus_hull": [ + [ + -0.01154593444489512, + -0.021689004533106276, + -0.0011374065314038427 + ], + [ + -0.016537690823465616, + -0.027227445437807935, + -0.0057864715503241106 + ], + [ + -0.014129673297373455, + -0.024669886454969016, + -0.0034380419597849097 + ], + [ + -0.012066011997219907, + -0.022183695083823802, + -0.0015022208890707429 + ] + ], + "cost_vs_hull": [ + [ + 0.11104680758615171, + 0.07828189469835561, + 0.14699484475552319 + ], + [ + 0.13607321514055926, + 0.10065338632691762, + 0.17507859415861599 + ], + [ + 0.11810433862788394, + 0.08327523872479879, + 0.1535466443546841 + ], + [ + 0.10847382209079726, + 0.07328610723408384, + 0.14558199254682896 + ] + ] + }, + "sc32": { + "acc_minus_hull": [ + [ + 0.01727890836704822, + 0.011744912763194603, + 0.023328553095136276 + ], + [ + 0.01272858047671425, + 0.007354553991362199, + 0.01836762727574271 + ], + [ + 0.014321840159119404, + 0.008847538807349205, + 0.020179350935957775 + ], + [ + 0.015851101629264938, + 0.010512041070992366, + 0.021600229987031472 + ] + ], + "cost_vs_hull": [ + [ + -0.14411166729135594, + -0.16652181663998855, + -0.11978340541668497 + ], + [ + -0.13656536882693682, + -0.16006792100782916, + -0.11255592836424118 + ], + [ + -0.13812488260488387, + -0.16274373031985748, + -0.11228978008622358 + ], + [ + -0.14773946465099874, + -0.17138060336285513, + -0.12408898797752163 + ] + ] + }, + "sc8": { + "acc_minus_hull": [ + [ + -0.011831013057281568, + -0.020924259595917337, + -0.0026943518330644727 + ], + [ + -0.014340175221638775, + -0.02380221186441814, + -0.005281672300869335 + ], + [ + -0.013505554275997738, + -0.022774420917686774, + -0.004476986827302682 + ], + [ + -0.011598439360796067, + -0.020457417145376758, + -0.002429287859219795 + ] + ], + "cost_vs_hull": [ + [ + 0.08470615401783421, + 0.052913132222845445, + 0.11952293446691922 + ], + [ + 0.10305505418229433, + 0.06846897513455145, + 0.14043684090919975 + ], + [ + 0.0836006068515962, + 0.051014442597240126, + 0.11869868317743142 + ], + [ + 0.08034059961749684, + 0.04859655962100912, + 0.11472260159294469 + ] + ] + }, + "scboth": { + "acc_minus_hull": [ + [ + 0.011381752078486577, + 0.006872236326157534, + 0.016268995972369034 + ], + [ + 0.00909360071205127, + 0.004674066807279793, + 0.01368502934783614 + ], + [ + 0.0095842394268741, + 0.005362710598833014, + 0.014148305465628867 + ], + [ + 0.008366983981449715, + 0.004355469257275526, + 0.012553652816093518 + ] + ], + "cost_vs_hull": [ + [ + -0.14431008697788583, + -0.17160719113740078, + -0.11579903905198777 + ], + [ + -0.1413480024591749, + -0.16755285404347794, + -0.11433735732866102 + ], + [ + -0.14260974935373127, + -0.1688997492213235, + -0.11537052361907747 + ], + [ + -0.14828294930936392, + -0.1749079374749139, + -0.12123062275791205 + ] + ] + }, + "scbud": { + "acc_minus_hull": [ + [ + -0.012763604027751434, + -0.022245581387111427, + -0.003001987229314552 + ], + [ + -0.013413936019365491, + -0.02257638835839211, + -0.0038743259030610527 + ], + [ + -0.016406413180826942, + -0.025928803882051342, + -0.006839534464497963 + ], + [ + -0.012481679714065406, + -0.02199072499730993, + -0.003176210068166959 + ] + ], + "cost_vs_hull": [ + [ + 0.08697793061074388, + 0.054973826539053486, + 0.12306320985625598 + ], + [ + 0.0658223665122728, + 0.03310762138813217, + 0.10214061336565375 + ], + [ + 0.08179331943576984, + 0.04991358266894612, + 0.11812508365989428 + ], + [ + 0.066359410448106, + 0.03170757660972136, + 0.1054058897793177 + ] + ] + } + } + }, + "gpu_s": { + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "hull": [ + { + "cost": 0.7290160193332379, + "acc": 0.7907788390889052, + "policy": "scboth:SC@1" + }, + { + "cost": 3.4543134172303263, + "acc": 0.929922850844967, + "policy": "scboth:agree2" + }, + { + "cost": 5.253141469447148, + "acc": 0.9478324761204996, + "policy": "scboth:always think" + }, + { + "cost": 13.76095541375684, + "acc": 0.9572005878030859, + "policy": "sc32:AC(0.95)+think@4" + }, + { + "cost": 20.678831458294123, + "acc": 0.9603232916972814, + "policy": "sc32:always think" + } + ], + "controllers": { + "bc8": { + "acc_minus_hull": [ + [ + -0.00048154355904439473, + -0.007730431883272332, + 0.006171427969939402 + ], + [ + 0.00048195462864539864, + -0.007078243540458368, + 0.008108301137940845 + ], + [ + 0.002284477163780532, + -0.004887972860546388, + 0.009277655654298604 + ], + [ + 0.003591728521452854, + -0.003270988758254005, + 0.010584169402185607 + ] + ], + "cost_vs_hull": [ + [ + 0.0032613595623585123, + -0.024845781466683484, + 0.0351182697593119 + ], + [ + -0.002783022760997511, + -0.034224052848013865, + 0.029567120765436414 + ], + [ + -0.03669478392009673, + -0.06359304530840613, + -0.0061231897176151715 + ], + [ + -0.0571685241224561, + -0.08379419009012114, + -0.02837579211958979 + ] + ] + }, + "sc32": { + "acc_minus_hull": [ + [ + -0.00760027306371133, + -0.014475690481403038, + -0.0008310043147514773 + ], + [ + -0.007782889804391352, + -0.015275805300946393, + -0.0006534257904663 + ], + [ + -0.009395025846732213, + -0.017491165745819545, + -0.0018847514330813963 + ], + [ + -0.009042885514006093, + -0.016713902476019588, + -0.001731084796915497 + ] + ], + "cost_vs_hull": [ + [ + 1.0735981614007977, + 0.9415138019372935, + 1.1988240443123437 + ], + [ + 0.9239211703026753, + 0.7907871694877033, + 1.0513820772074665 + ], + [ + 0.9767319261805385, + 0.8434704144697326, + 1.1047462004883983 + ], + [ + 1.082060399531183, + 0.9467845448576931, + 1.2129171792397617 + ] + ] + }, + "sc8": { + "acc_minus_hull": [ + [ + -0.0016858458055734418, + -0.007151926978278105, + 0.003893793354212172 + ], + [ + 5.592856338770559e-05, + -0.0054508032955559995, + 0.005653126710551691 + ], + [ + -0.002868407827873601, + -0.008516665206462699, + 0.0030549398174657947 + ], + [ + -0.0018096210435353628, + -0.007758177437949999, + 0.004416730344314157 + ] + ], + "cost_vs_hull": [ + [ + 0.01589699109308862, + -0.014653279134561844, + 0.04950279856017859 + ], + [ + 0.021314558716862164, + -0.011253524611173188, + 0.05615939583292788 + ], + [ + 0.013562872849064211, + -0.0177623173158587, + 0.047546722784043274 + ], + [ + 0.01039326387117967, + -0.020183755015209477, + 0.04340494874304136 + ] + ] + }, + "scboth": { + "acc_minus_hull": [ + [ + -0.0016858458055734418, + -0.007151926978278105, + 0.003893793354212172 + ], + [ + 5.592856338770559e-05, + -0.0054508032955559995, + 0.005653126710551691 + ], + [ + -0.002868407827873601, + -0.008516665206462699, + 0.0030549398174657947 + ], + [ + -0.0018096210435353628, + -0.007758177437949999, + 0.004416730344314157 + ] + ], + "cost_vs_hull": [ + [ + 0.01589699109308862, + -0.014653279134561844, + 0.04950279856017859 + ], + [ + 0.021314558716862164, + -0.011253524611173188, + 0.05615939583292788 + ], + [ + 0.013562872849064211, + -0.0177623173158587, + 0.047546722784043274 + ], + [ + 0.01039326387117967, + -0.020183755015209477, + 0.04340494874304136 + ] + ] + }, + "scbud": { + "acc_minus_hull": [ + [ + -0.001975094951340463, + -0.008351364144323446, + 0.004671268656209748 + ], + [ + 0.00397995179486257, + -0.004830132768530077, + 0.012795608565363424 + ], + [ + -0.0015036984082572224, + -0.00749028953020849, + 0.004561025654270194 + ], + [ + -0.0015299221051515927, + -0.007715180846166277, + 0.0044897653115898006 + ] + ], + "cost_vs_hull": [ + [ + 0.010390459635348348, + -0.020932530159203898, + 0.04403831969254968 + ], + [ + 0.012379209238011946, + -0.01880095002203893, + 0.04707347028879613 + ], + [ + 0.01976795325589742, + -0.011908985163736324, + 0.05347997084323053 + ], + [ + 0.003533757204437382, + -0.0291597789461571, + 0.04147560355249579 + ] + ] + } + } + }, + "usd_cheapest": { + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "hull": [ + { + "cost": 0.00022992130786186627, + "acc": 0.7907788390889052, + "policy": "sc8:SC@1" + }, + { + "cost": 0.0009421097997795738, + "acc": 0.929922850844967, + "policy": "sc8:agree2" + }, + { + "cost": 0.0010301091219691403, + "acc": 0.9405767817781043, + "policy": "sc32:agree2" + }, + { + "cost": 0.0014912116385011021, + "acc": 0.9603232916972814, + "policy": "sc32:always think" + } + ], + "controllers": { + "bc8": { + "acc_minus_hull": [ + [ + 0.006461853195983721, + 0.00023723940444089547, + 0.012824841407981088 + ], + [ + 0.0031895237842326334, + -0.003773076827493333, + 0.010254499139280114 + ], + [ + 0.0060612350213929776, + -0.0005033765276781783, + 0.012814469917665779 + ], + [ + 0.0060869710999496875, + -0.0004956335967164088, + 0.012472244540277557 + ] + ], + "cost_vs_hull": [ + [ + -0.0532063450098913, + -0.08032570572421102, + -0.022156298094874495 + ], + [ + -0.023445205834801475, + -0.05121628786760856, + 0.007151708436422067 + ], + [ + -0.044406585382604336, + -0.07245505214595364, + -0.014705170323175008 + ], + [ + -0.04911023839032702, + -0.07746816257675937, + -0.018718227898915377 + ] + ] + }, + "sc32": { + "acc_minus_hull": [ + [ + 0.00847387069304179, + 0.002752211932109202, + 0.0147067869511535 + ], + [ + 0.00543196119618611, + -0.0007901506437870733, + 0.012287642868848895 + ], + [ + 0.00717665576660485, + 0.0012001277069492444, + 0.013546023587525744 + ], + [ + 0.00899974445953801, + 0.0032905861335968184, + 0.015207017410604217 + ] + ], + "cost_vs_hull": [ + [ + -0.05146112650207901, + -0.08056158073637068, + -0.020214674567855802 + ], + [ + -0.05149310055490697, + -0.08117172461135748, + -0.020903399342985136 + ], + [ + -0.04031006638550416, + -0.07139890154140159, + -0.007567716145226131 + ], + [ + -0.059042036571112066, + -0.08773568656607754, + -0.02924173278476002 + ] + ] + }, + "sc8": { + "acc_minus_hull": [ + [ + 0.007576206175890477, + 0.0025087311362865395, + 0.013045562464691556 + ], + [ + 0.003766617344983701, + -0.0013999603117203906, + 0.009214176870289892 + ], + [ + 0.004563308966810569, + -0.0004414767163326162, + 0.009982355994823666 + ], + [ + 0.007148919768599523, + 0.0021915383390930733, + 0.012599007002087456 + ] + ], + "cost_vs_hull": [ + [ + -0.06543636536668762, + -0.09135087534327356, + -0.037435888101107796 + ], + [ + -0.04401704481886837, + -0.07284031632815699, + -0.014262590641537556 + ], + [ + -0.06380618549466399, + -0.08943566919345203, + -0.03604802468648296 + ], + [ + -0.06736791249423935, + -0.09356042224189386, + -0.03959883576869828 + ] + ] + }, + "scboth": { + "acc_minus_hull": [ + [ + 0.005947591240978716, + -0.0025791257593269336, + 0.014957702081186372 + ], + [ + -0.002584385097995378, + -0.009911241786641621, + 0.005086326768490827 + ], + [ + 0.004981732806636208, + -0.0029627403610281494, + 0.013296864663134305 + ], + [ + 0.008135585203617056, + -0.0001171314641442045, + 0.0168435688710609 + ] + ], + "cost_vs_hull": [ + [ + -0.04843912481007129, + -0.08004102905532201, + -0.01546400925358868 + ], + [ + 0.0013963669178986837, + -0.033358295836115555, + 0.037145039593570066 + ], + [ + -0.039138578334908636, + -0.07146530699156732, + -0.006454509662151425 + ], + [ + -0.039401684670443915, + -0.07101248647668243, + -0.007422798030194643 + ] + ] + }, + "scbud": { + "acc_minus_hull": [ + [ + 0.007533829413569287, + 0.001341474908743934, + 0.013617621373855814 + ], + [ + 0.004074135588326202, + -0.002015959278434806, + 0.010195952000522916 + ], + [ + 0.0024343497149646575, + -0.0035408088080407314, + 0.0084421258722923 + ], + [ + 0.006984600043258462, + 0.0008704167515859184, + 0.012992727213065788 + ] + ], + "cost_vs_hull": [ + [ + -0.05720382831094939, + -0.08417090680475348, + -0.028265662067081195 + ], + [ + -0.07865556268370588, + -0.10500652075273233, + -0.049304273592419914 + ], + [ + -0.06516232712183578, + -0.0917846688165415, + -0.036465432932704704 + ], + [ + -0.07342046969120042, + -0.10118522293955877, + -0.04124073904680408 + ] + ] + } + } + }, + "usd_openrouter_only": { + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "hull": [ + { + "cost": 0.000523391629316679, + "acc": 0.7907788390889052, + "policy": "sc8:SC@1" + }, + { + "cost": 0.0014912116385011026, + "acc": 0.9603232916972814, + "policy": "sc32:always think" + } + ], + "controllers": { + "bc8": { + "acc_minus_hull": [ + [ + -0.024089430040936284, + -0.035795633462789966, + -0.011859714495643965 + ], + [ + -0.02844547076729296, + -0.0403589797417865, + -0.015528235541093768 + ], + [ + -0.026503621286868898, + -0.03831216542458276, + -0.013800251915608285 + ], + [ + -0.024876666316783876, + -0.03679279941219691, + -0.012332056261152519 + ] + ], + "cost_vs_hull": [ + [ + 0.5812175652220934, + 0.5243003483552899, + 0.6372331591782454 + ], + [ + 0.6058483956040768, + 0.5476871666208383, + 0.6624092921113377 + ], + [ + 0.5811971521971844, + 0.5232408833209826, + 0.6382887495144551 + ], + [ + 0.5797636910122075, + 0.5195947649171093, + 0.6381106633300871 + ] + ] + }, + "sc32": { + "acc_minus_hull": [ + [ + -0.002387950036737693, + -0.007255694342395298, + 0.002387950036737693 + ], + [ + -0.002204261572373255, + -0.00615356355620867, + 0.002204261572373255 + ], + [ + -0.0017450404114621601, + -0.00615356355620867, + 0.002571638501102131 + ], + [ + -0.0033063923585598823, + -0.00707200587803086, + 0.0002755326965466569 + ] + ], + "cost_vs_hull": [ + [ + -0.15043938841044702, + -0.1744256228639952, + -0.12680480574523081 + ], + [ + -0.15858652125182837, + -0.18102165272068413, + -0.1370119177141602 + ], + [ + -0.15779677837051487, + -0.18188544779191038, + -0.13637997604005572 + ], + [ + -0.14605338799686296, + -0.16856229733308767, + -0.12426933629383971 + ] + ] + }, + "sc8": { + "acc_minus_hull": [ + [ + -0.024476487876561353, + -0.034993800514327704, + -0.013499954077883914 + ], + [ + -0.025165319617927995, + -0.036188923585598824, + -0.014234707935341666 + ], + [ + -0.025349008082292433, + -0.036095931300514326, + -0.014188785819250557 + ], + [ + -0.025303085966201323, + -0.035775624540778844, + -0.014096941587068338 + ] + ], + "cost_vs_hull": [ + [ + 0.5330814208554671, + 0.47909890146732326, + 0.5880885982065726 + ], + [ + 0.5540628489231623, + 0.4993388975245331, + 0.6099124561612825 + ], + [ + 0.5305390807229236, + 0.47670982007855967, + 0.5854206884726549 + ], + [ + 0.5250235396658292, + 0.472202835658319, + 0.5776393799507212 + ] + ] + }, + "scboth": { + "acc_minus_hull": [ + [ + -0.002709404849375459, + -0.009047804922850844, + 0.003719691403379868 + ], + [ + -0.002020573108008817, + -0.006612784717119765, + 0.002755326965466569 + ], + [ + -0.0005510653930933138, + -0.0048677443056576045, + 0.004132990448199853 + ], + [ + -0.0022501836884643643, + -0.006109937545922116, + 0.0016072740631888317 + ] + ], + "cost_vs_hull": [ + [ + -0.16955905817011319, + -0.19479604305419604, + -0.14498247995196106 + ], + [ + -0.16504706885748432, + -0.18956744646922988, + -0.14073116481625095 + ], + [ + -0.16789743150206637, + -0.19399132077984557, + -0.14440360324613755 + ], + [ + -0.17356432799292065, + -0.19917782540984438, + -0.15050376803231216 + ] + ] + }, + "scbud": { + "acc_minus_hull": [ + [ + -0.025716385011021307, + -0.03710966201322557, + -0.013868479059515063 + ], + [ + -0.024017266715650257, + -0.03518093313739897, + -0.012674504041146216 + ], + [ + -0.02902277736958119, + -0.040135929463629684, + -0.017266715650257163 + ], + [ + -0.026083761939750184, + -0.03692367744305658, + -0.014600936811168272 + ] + ], + "cost_vs_hull": [ + [ + 0.5266090507648056, + 0.4757842841987392, + 0.5791280535767499 + ], + [ + 0.5020126128419893, + 0.45163260264468635, + 0.5535362852806782 + ], + [ + 0.5196884944831792, + 0.4685816649085815, + 0.5712519835354636 + ], + [ + 0.48981611738438446, + 0.44149995600214287, + 0.540441658966268 + ] + ] + } + } + }, + "usd_dear_32b": { + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "hull": [ + { + "cost": 0.00022992130786186627, + "acc": 0.7907788390889052, + "policy": "sc8:SC@1" + }, + { + "cost": 0.0009421097997795738, + "acc": 0.929922850844967, + "policy": "sc8:agree2" + }, + { + "cost": 0.0012692386480529024, + "acc": 0.9478324761204996, + "policy": "sc8:always think" + }, + { + "cost": 0.002509960915686995, + "acc": 0.9572005878030859, + "policy": "sc32:AC(0.95)+think@4" + }, + { + "cost": 0.0030334256869948566, + "acc": 0.9603232916972814, + "policy": "sc32:always think" + } + ], + "controllers": { + "bc8": { + "acc_minus_hull": [ + [ + 0.006461853195983721, + 0.00023723940444089547, + 0.012824841407981088 + ], + [ + 0.0031895237842326334, + -0.003773076827493333, + 0.010254499139280114 + ], + [ + 0.0060612350213929776, + -0.0005033765276781783, + 0.012814469917665779 + ], + [ + 0.0060869710999496875, + -0.0004956335967164088, + 0.012472244540277557 + ] + ], + "cost_vs_hull": [ + [ + -0.10974827000647636, + -0.13164471872184877, + -0.08750871203781087 + ], + [ + -0.042980839821979866, + -0.0677209485479761, + -0.015696041542626302 + ], + [ + -0.08117710959093671, + -0.10381168964096052, + -0.05735558323834158 + ], + [ + -0.09949844579497735, + -0.12349471610974982, + -0.07626208276844426 + ] + ] + }, + "sc32": { + "acc_minus_hull": [ + [ + -0.011346723989888419, + -0.019964106033219495, + -0.002960933184705086 + ], + [ + -0.010059606225570629, + -0.019071796604454058, + -0.0016658928326801245 + ], + [ + -0.008972906420464779, + -0.017919818285893632, + -0.00038094429517570525 + ], + [ + -0.009798900391291632, + -0.018426240185688145, + -0.001526370165657511 + ] + ], + "cost_vs_hull": [ + [ + 0.4743101864247081, + 0.40188290308329594, + 0.545478444534539 + ], + [ + 0.4162609787291198, + 0.3502323470544917, + 0.4780863209115802 + ], + [ + 0.4342922124038393, + 0.3624034408695393, + 0.5034776263996422 + ], + [ + 0.43488201009475924, + 0.36296893772890954, + 0.5039562088447429 + ] + ] + }, + "sc8": { + "acc_minus_hull": [ + [ + 0.007576206175890477, + 0.0025087311362865395, + 0.013045562464691556 + ], + [ + 0.004520742121220974, + -0.00042030120219384076, + 0.009825390864408734 + ], + [ + 0.004563308966810569, + -0.0004414767163326162, + 0.009982355994823666 + ], + [ + 0.007148919768599523, + 0.0021915383390930733, + 0.012599007002087456 + ] + ], + "cost_vs_hull": [ + [ + -0.08346617864261463, + -0.10673399083793, + -0.058709905582825926 + ], + [ + -0.04401704481886837, + -0.07284031632815699, + -0.014262590641537556 + ], + [ + -0.07453530543720732, + -0.09899771618904023, + -0.04877543454532804 + ], + [ + -0.07849186281833131, + -0.10345349813277324, + -0.05257902379449454 + ] + ] + }, + "scboth": { + "acc_minus_hull": [ + [ + 0.007576206175890477, + 0.0025087311362865395, + 0.013045562464691556 + ], + [ + 0.004520742121220974, + -0.00042030120219384076, + 0.009825390864408734 + ], + [ + 0.004563308966810569, + -0.0004414767163326162, + 0.009982355994823666 + ], + [ + 0.007148919768599523, + 0.0021915383390930733, + 0.012599007002087456 + ] + ], + "cost_vs_hull": [ + [ + -0.08346617864261463, + -0.10673399083793, + -0.058709905582825926 + ], + [ + -0.04401704481886837, + -0.07284031632815699, + -0.014262590641537556 + ], + [ + -0.07453530543720732, + -0.09899771618904023, + -0.04877543454532804 + ], + [ + -0.07849186281833131, + -0.10345349813277324, + -0.05257902379449454 + ] + ] + }, + "scbud": { + "acc_minus_hull": [ + [ + 0.007533829413569287, + 0.001341474908743934, + 0.013617621373855814 + ], + [ + 0.004936518010892989, + -0.0010581317615102007, + 0.010792463014637826 + ], + [ + 0.0024343497149646575, + -0.0035408088080407314, + 0.0084421258722923 + ], + [ + 0.006984600043258462, + 0.0008704167515859184, + 0.012992727213065788 + ] + ], + "cost_vs_hull": [ + [ + -0.05720382831094939, + -0.08417090680475348, + -0.028265662067081195 + ], + [ + -0.0831094764116378, + -0.10884936074165545, + -0.05485464715086791 + ], + [ + -0.06516232712183578, + -0.0917846688165415, + -0.036465432932704704 + ], + [ + -0.07342046969120042, + -0.10118522293955877, + -0.04124073904680408 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/currency_table.log b/analysis/bayes_sc_2026-10-25/v8/currency_table.log new file mode 100644 index 0000000000000000000000000000000000000000..eafb729b39f22f7336ef2435ac1c8df139327a65 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/currency_table.log @@ -0,0 +1,55 @@ +== tokens (base tokens): hull SC@1 0.791@1125 -> agree2 0.941@4284 -> always think 0.960@5317 + bc8 acc - hull: -1.2 [-2.2, -0.1] | -1.7 [-2.7, -0.6] | -1.4 [-2.5, -0.3] | -1.2 [-2.2, -0.2] + cost vs hull at its acc: +11.1% [+7.8, +14.7] | +13.6% [+10.1, +17.5] | +11.8% [+8.3, +15.4] | +10.8% [+7.3, +14.6] + sc32 acc - hull: +1.7 [+1.2, +2.3] | +1.3 [+0.7, +1.8] | +1.4 [+0.9, +2.0] | +1.6 [+1.1, +2.2] + cost vs hull at its acc: -14.4% [-16.7, -12.0] | -13.7% [-16.0, -11.3] | -13.8% [-16.3, -11.2] | -14.8% [-17.1, -12.4] + sc8 acc - hull: -1.2 [-2.1, -0.3] | -1.4 [-2.4, -0.5] | -1.4 [-2.3, -0.4] | -1.2 [-2.0, -0.2] + cost vs hull at its acc: +8.5% [+5.3, +12.0] | +10.3% [+6.8, +14.0] | +8.4% [+5.1, +11.9] | +8.0% [+4.9, +11.5] + scboth acc - hull: +1.1 [+0.7, +1.6] | +0.9 [+0.5, +1.4] | +1.0 [+0.5, +1.4] | +0.8 [+0.4, +1.3] + cost vs hull at its acc: -14.4% [-17.2, -11.6] | -14.1% [-16.8, -11.4] | -14.3% [-16.9, -11.5] | -14.8% [-17.5, -12.1] + scbud acc - hull: -1.3 [-2.2, -0.3] | -1.3 [-2.3, -0.4] | -1.6 [-2.6, -0.7] | -1.2 [-2.2, -0.3] + cost vs hull at its acc: +8.7% [+5.5, +12.3] | +6.6% [+3.3, +10.2] | +8.2% [+5.0, +11.8] | +6.6% [+3.2, +10.5] +== gpu_s (GPU-seconds): hull SC@1 0.791@0.729 -> agree2 0.930@3.454 -> always think 0.948@5.253 -> AC(0.95)+think@4 0.957@13.76 -> always think 0.960@20.68 + bc8 acc - hull: -0.0 [-0.8, +0.6] | +0.0 [-0.7, +0.8] | +0.2 [-0.5, +0.9] | +0.4 [-0.3, +1.1] + cost vs hull at its acc: +0.3% [-2.5, +3.5] | -0.3% [-3.4, +3.0] | -3.7% [-6.4, -0.6] | -5.7% [-8.4, -2.8] + sc32 acc - hull: -0.8 [-1.4, -0.1] | -0.8 [-1.5, -0.1] | -0.9 [-1.7, -0.2] | -0.9 [-1.7, -0.2] + cost vs hull at its acc: +107.4% [+94.2, +119.9] | +92.4% [+79.1, +105.1] | +97.7% [+84.3, +110.5] | +108.2% [+94.7, +121.3] + sc8 acc - hull: -0.2 [-0.7, +0.4] | +0.0 [-0.5, +0.6] | -0.3 [-0.9, +0.3] | -0.2 [-0.8, +0.4] + cost vs hull at its acc: +1.6% [-1.5, +5.0] | +2.1% [-1.1, +5.6] | +1.4% [-1.8, +4.8] | +1.0% [-2.0, +4.3] + scboth acc - hull: -0.2 [-0.7, +0.4] | +0.0 [-0.5, +0.6] | -0.3 [-0.9, +0.3] | -0.2 [-0.8, +0.4] + cost vs hull at its acc: +1.6% [-1.5, +5.0] | +2.1% [-1.1, +5.6] | +1.4% [-1.8, +4.8] | +1.0% [-2.0, +4.3] + scbud acc - hull: -0.2 [-0.8, +0.5] | +0.4 [-0.5, +1.3] | -0.2 [-0.7, +0.5] | -0.2 [-0.8, +0.4] + cost vs hull at its acc: +1.0% [-2.1, +4.4] | +1.2% [-1.9, +4.7] | +2.0% [-1.2, +5.3] | +0.4% [-2.9, +4.1] +== usd_cheapest (USD): hull SC@1 0.791@0.0002299 -> agree2 0.930@0.0009421 -> agree2 0.941@0.00103 -> always think 0.960@0.001491 + bc8 acc - hull: +0.6 [+0.0, +1.3] | +0.3 [-0.4, +1.0] | +0.6 [-0.1, +1.3] | +0.6 [-0.0, +1.2] + cost vs hull at its acc: -5.3% [-8.0, -2.2] | -2.3% [-5.1, +0.7] | -4.4% [-7.2, -1.5] | -4.9% [-7.7, -1.9] + sc32 acc - hull: +0.8 [+0.3, +1.5] | +0.5 [-0.1, +1.2] | +0.7 [+0.1, +1.4] | +0.9 [+0.3, +1.5] + cost vs hull at its acc: -5.1% [-8.1, -2.0] | -5.1% [-8.1, -2.1] | -4.0% [-7.1, -0.8] | -5.9% [-8.8, -2.9] + sc8 acc - hull: +0.8 [+0.3, +1.3] | +0.4 [-0.1, +0.9] | +0.5 [-0.0, +1.0] | +0.7 [+0.2, +1.3] + cost vs hull at its acc: -6.5% [-9.1, -3.7] | -4.4% [-7.3, -1.4] | -6.4% [-8.9, -3.6] | -6.7% [-9.4, -4.0] + scboth acc - hull: +0.6 [-0.3, +1.5] | -0.3 [-1.0, +0.5] | +0.5 [-0.3, +1.3] | +0.8 [-0.0, +1.7] + cost vs hull at its acc: -4.8% [-8.0, -1.5] | +0.1% [-3.3, +3.7] | -3.9% [-7.1, -0.6] | -3.9% [-7.1, -0.7] + scbud acc - hull: +0.8 [+0.1, +1.4] | +0.4 [-0.2, +1.0] | +0.2 [-0.4, +0.8] | +0.7 [+0.1, +1.3] + cost vs hull at its acc: -5.7% [-8.4, -2.8] | -7.9% [-10.5, -4.9] | -6.5% [-9.2, -3.6] | -7.3% [-10.1, -4.1] +== usd_openrouter_only (USD): hull SC@1 0.791@0.0005234 -> always think 0.960@0.001491 + bc8 acc - hull: -2.4 [-3.6, -1.2] | -2.8 [-4.0, -1.6] | -2.7 [-3.8, -1.4] | -2.5 [-3.7, -1.2] + cost vs hull at its acc: +58.1% [+52.4, +63.7] | +60.6% [+54.8, +66.2] | +58.1% [+52.3, +63.8] | +58.0% [+52.0, +63.8] + sc32 acc - hull: -0.2 [-0.7, +0.2] | -0.2 [-0.6, +0.2] | -0.2 [-0.6, +0.3] | -0.3 [-0.7, +0.0] + cost vs hull at its acc: -15.0% [-17.4, -12.7] | -15.9% [-18.1, -13.7] | -15.8% [-18.2, -13.6] | -14.6% [-16.9, -12.4] + sc8 acc - hull: -2.4 [-3.5, -1.3] | -2.5 [-3.6, -1.4] | -2.5 [-3.6, -1.4] | -2.5 [-3.6, -1.4] + cost vs hull at its acc: +53.3% [+47.9, +58.8] | +55.4% [+49.9, +61.0] | +53.1% [+47.7, +58.5] | +52.5% [+47.2, +57.8] + scboth acc - hull: -0.3 [-0.9, +0.4] | -0.2 [-0.7, +0.3] | -0.1 [-0.5, +0.4] | -0.2 [-0.6, +0.2] + cost vs hull at its acc: -17.0% [-19.5, -14.5] | -16.5% [-19.0, -14.1] | -16.8% [-19.4, -14.4] | -17.4% [-19.9, -15.1] + scbud acc - hull: -2.6 [-3.7, -1.4] | -2.4 [-3.5, -1.3] | -2.9 [-4.0, -1.7] | -2.6 [-3.7, -1.5] + cost vs hull at its acc: +52.7% [+47.6, +57.9] | +50.2% [+45.2, +55.4] | +52.0% [+46.9, +57.1] | +49.0% [+44.1, +54.0] +== usd_dear_32b (USD): hull SC@1 0.791@0.0002299 -> agree2 0.930@0.0009421 -> always think 0.948@0.001269 -> AC(0.95)+think@4 0.957@0.00251 -> always think 0.960@0.003033 + bc8 acc - hull: +0.6 [+0.0, +1.3] | +0.3 [-0.4, +1.0] | +0.6 [-0.1, +1.3] | +0.6 [-0.0, +1.2] + cost vs hull at its acc: -11.0% [-13.2, -8.8] | -4.3% [-6.8, -1.6] | -8.1% [-10.4, -5.7] | -9.9% [-12.3, -7.6] + sc32 acc - hull: -1.1 [-2.0, -0.3] | -1.0 [-1.9, -0.2] | -0.9 [-1.8, -0.0] | -1.0 [-1.8, -0.2] + cost vs hull at its acc: +47.4% [+40.2, +54.5] | +41.6% [+35.0, +47.8] | +43.4% [+36.2, +50.3] | +43.5% [+36.3, +50.4] + sc8 acc - hull: +0.8 [+0.3, +1.3] | +0.5 [-0.0, +1.0] | +0.5 [-0.0, +1.0] | +0.7 [+0.2, +1.3] + cost vs hull at its acc: -8.3% [-10.7, -5.9] | -4.4% [-7.3, -1.4] | -7.5% [-9.9, -4.9] | -7.8% [-10.3, -5.3] + scboth acc - hull: +0.8 [+0.3, +1.3] | +0.5 [-0.0, +1.0] | +0.5 [-0.0, +1.0] | +0.7 [+0.2, +1.3] + cost vs hull at its acc: -8.3% [-10.7, -5.9] | -4.4% [-7.3, -1.4] | -7.5% [-9.9, -4.9] | -7.8% [-10.3, -5.3] + scbud acc - hull: +0.8 [+0.1, +1.4] | +0.5 [-0.1, +1.1] | +0.2 [-0.4, +0.8] | +0.7 [+0.1, +1.3] + cost vs hull at its acc: -5.7% [-8.4, -2.8] | -8.3% [-10.9, -5.5] | -6.5% [-9.2, -3.6] | -7.3% [-10.1, -4.1] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..2c23ba6d386b086a9ce61e5797b06928062306fd --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b10e6addd10ff1db6b8ebd9780f33451dcc3526c8254a5d3e43c3fabd8cfe34 +size 325078 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.json b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.json new file mode 100644 index 0000000000000000000000000000000000000000..329e9365a305310fddb2afacc8664904b9a0dbc1 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.json @@ -0,0 +1,490 @@ +{ + "ref": { + "acc": 0.9405767817781043, + "equiv": 14776.582078233669 + }, + "cost_model": { + "currency": "gpu_s", + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.9405767817781043, + "equiv": 14776.582078233669, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6582624885161, + "delta_vs_agree2": [ + -0.14979794268919913, + -0.16256429096252756, + -0.1369374540778839 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.3165249770323, + "delta_vs_agree2": [ + -0.12876561351947097, + -0.14226671565025717, + -0.11544819985304923 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.9747874655477, + "delta_vs_agree2": [ + -0.09616091109478325, + -0.10911324393828067, + -0.08357825128581925 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.6330499540645, + "delta_vs_agree2": [ + -0.0852314474650992, + -0.09809652828802351, + -0.07274063188831742 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.29131244258, + "delta_vs_agree2": [ + -0.0775165319617928, + -0.09056070903747244, + -0.06493387215282881 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.949574931095, + "delta_vs_agree2": [ + -0.07127112417340191, + -0.08440714548126378, + -0.05905584129316679 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.607837419612, + "delta_vs_agree2": [ + -0.068791329904482, + -0.08174136664217486, + -0.05648190668626012 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9197.266099908129, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4412.0116364692885, + "delta_vs_agree2": [ + -0.06915870683321088, + -0.08238886847905952, + -0.05639235855988244 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.9405767817781043, + "equiv": 14776.582078233669, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9403930933137399, + "equiv": 15250.10020965791, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9340558412931668, + "equiv": 12428.038051228672, + "delta_vs_agree2": [ + -0.006520940484937546, + -0.009184423218221896, + -0.004132990448199853 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.396916034446, + "delta_vs_agree2": [ + -0.06484202792064658, + -0.07834542615723733, + -0.05198383541513593 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9603232916972814, + "equiv": 34909.83373470687, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9542615723732549, + "equiv": 18891.999423466008, + "delta_vs_agree2": [ + 0.013684790595150624, + 0.010929463629684055, + 0.016623806024981633 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9540778839088905, + "equiv": 19471.496539687407, + "delta_vs_agree2": [ + 0.013501102130786187, + 0.010745775165319618, + 0.0162587252020573 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.669323298678, + "delta_vs_agree2": [ + -0.06456649522409992, + -0.07806989346069067, + -0.05152461425422483 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9603232916972814, + "equiv": 34909.83373470687, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9603232916972814, + "equiv": 36059.49199719538, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9572005878030859, + "equiv": 21701.026687494537, + "delta_vs_agree2": [ + 0.016623806024981633, + 0.012950036737692874, + 0.020575404114621592 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.71692481839, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9603232916972814, + "equiv": 34909.83373470687, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9603232916972814, + "equiv": 36059.49199719538, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9603232916972814, + "equiv": 37209.1502596839, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "always think": { + "acc": 0.9603232916972814, + "equiv": 32610.51720972983, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 16000, + "acc": 0.8075863335782513, + "equiv": 1396.4326338052047 + }, + { + "R": 24000, + "acc": 0.8096987509184423, + "equiv": 1472.383530754564 + }, + { + "R": 32000, + "acc": 0.8292615723732549, + "equiv": 2379.59416181811 + }, + { + "R": 48000, + "acc": 0.863243938280676, + "equiv": 3899.246677291924 + }, + { + "R": 64000, + "acc": 0.8943791329904482, + "equiv": 7482.135276499471 + }, + { + "R": 96000, + "acc": 0.9239529757531227, + "equiv": 11234.58117973906 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9448016164584864, + "equiv": 14828.426778364617, + "R": [ + 209675.72260947726, + 368276.6581960034 + ], + "delta_vs_agree2": [ + 0.004224834680382072, + 0.0001836884643644379, + 0.008357825128581926 + ], + "actions": { + "stop@2": 1991, + "stop@3": 4195, + "stop@5": 1092, + "stop@6": 376, + "stop@4": 950, + "think:q32b_think@8": 624, + "think:q32b_think@7": 208, + "think:q32b_think@6": 220, + "stop@7": 263, + "think:q32b_think@3": 247, + "think:q32b_think@5": 366, + "stop@8": 218, + "think:q32b_think@4": 137, + "think:q32b_think@2": 1 + } + }, + { + "seed": 1, + "acc": 0.9444342395297576, + "equiv": 14563.816924343886, + "R": [ + 267815.15620037477, + 295919.7331034693 + ], + "delta_vs_agree2": [ + 0.003857457751653196, + -0.0004592211609110948, + 0.008174136664217487 + ], + "actions": { + "stop@8": 240, + "stop@7": 297, + "stop@6": 334, + "stop@5": 910, + "stop@3": 2091, + "think:q32b_think@8": 520, + "stop@2": 3447, + "stop@4": 1802, + "think:q32b_think@3": 133, + "think:q32b_think@4": 396, + "think:q32b_think@2": 5, + "think:q32b_think@5": 304, + "think:q32b_think@7": 179, + "think:q32b_think@6": 230 + } + }, + { + "seed": 2, + "acc": 0.9426891991182954, + "equiv": 14373.47393929234, + "R": [ + 250752.31351567752, + 288676.0527428261 + ], + "delta_vs_agree2": [ + 0.002112417340191036, + -0.002387950036737693, + 0.006337252020573108 + ], + "actions": { + "stop@5": 1013, + "stop@8": 229, + "stop@7": 232, + "stop@6": 331, + "stop@4": 1435, + "stop@3": 2894, + "stop@2": 2997, + "think:q32b_think@6": 391, + "think:q32b_think@3": 118, + "think:q32b_think@5": 261, + "think:q32b_think@8": 447, + "think:q32b_think@4": 397, + "think:q32b_think@7": 140, + "think:q32b_think@2": 3 + } + }, + { + "seed": 3, + "acc": 0.9433321087435709, + "equiv": 14789.907917129956, + "R": [ + 297679.4449697336, + 308943.665263941 + ], + "delta_vs_agree2": [ + 0.002755326965466569, + -0.002112417340191036, + 0.007531227038941954 + ], + "actions": { + "stop@2": 3282, + "stop@3": 2624, + "stop@8": 199, + "stop@7": 275, + "stop@6": 403, + "stop@4": 1575, + "stop@5": 756, + "think:q32b_think@7": 249, + "think:q32b_think@5": 359, + "think:q32b_think@6": 212, + "think:q32b_think@4": 290, + "think:q32b_think@8": 661, + "think:q32b_think@2": 3 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9356171932402645, + "equiv": 13166.091453002167, + "R": [ + 122453.90315791618, + 215367.45691742294 + ], + "delta_acc": [ + -0.004959588537839824, + -0.009735488611315209, + -9.184423218221895e-05 + ], + "rel_cost": [ + -0.10898938717389867, + -0.14274393354401801, + -0.07372849351896534 + ] + }, + { + "seed": 1, + "acc": 0.9377296105804556, + "equiv": 12859.460458165413, + "R": [ + 128763.81386976562, + 142514.1213197139 + ], + "delta_acc": [ + -0.002847171197648788, + -0.007531227038941954, + 0.0018368846436443791 + ], + "rel_cost": [ + -0.12974053200653404, + -0.16560395904727204, + -0.09532283731871877 + ] + }, + { + "seed": 2, + "acc": 0.9364437913299045, + "equiv": 12809.857169532603, + "R": [ + 122551.04267303023, + 142163.9738900473 + ], + "delta_acc": [ + -0.004132990448199853, + -0.009186719324026451, + 0.0008265980896399706 + ], + "rel_cost": [ + -0.13309741713532697, + -0.1681467862102402, + -0.09717950555075175 + ] + }, + { + "seed": 3, + "acc": 0.9346069066862601, + "equiv": 12886.646277828833, + "R": [ + 171552.75460171435, + 130081.31657895725 + ], + "delta_acc": [ + -0.005969875091844232, + -0.011572373254959589, + -0.0005510653930933138 + ], + "rel_cost": [ + -0.12790074121327172, + -0.16479919562013812, + -0.08909805666181518 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.log b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.log new file mode 100644 index 0000000000000000000000000000000000000000..59a887358325629e4858e812f804011529c71940 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,490 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> q32b_think): acc 0.941 equiv 14777 +cost model gpu_s: 1 equiv = 0.0006341 GPU-seconds; agree2 costs 9.37 GPU-seconds per problem + agree2 acc 0.941 equiv 14777 ( +0; 9.37 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 (-13627; 0.729 GPU-seconds) | vs agree2 -15.0 [-16.3, -13.7] + SC@2 acc 0.812 equiv 2299 (-12477; 1.458 GPU-seconds) | vs agree2 -12.9 [-14.2, -11.5] + SC@3 acc 0.844 equiv 3449 (-11328; 2.187 GPU-seconds) | vs agree2 -9.6 [-10.9, -8.4] + SC@4 acc 0.855 equiv 4599 (-10178; 2.916 GPU-seconds) | vs agree2 -8.5 [-9.8, -7.3] + SC@5 acc 0.863 equiv 5748 ( -9028; 3.645 GPU-seconds) | vs agree2 -7.8 [-9.1, -6.5] + SC@6 acc 0.869 equiv 6898 ( -7879; 4.374 GPU-seconds) | vs agree2 -7.1 [-8.4, -5.9] + SC@7 acc 0.872 equiv 8048 ( -6729; 5.103 GPU-seconds) | vs agree2 -6.9 [-8.2, -5.6] + SC@8 acc 0.876 equiv 9197 ( -5579; 5.832 GPU-seconds) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.8) acc 0.871 equiv 4412 (-10365; 2.798 GPU-seconds) | vs agree2 -6.9 [-8.2, -5.6] + AC(0.8)+think@2 acc 0.941 equiv 14777 ( +0; 9.37 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.940 equiv 15250 ( +474; 9.67 GPU-seconds) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.934 equiv 12428 ( -2349; 7.881 GPU-seconds) | vs agree2 -0.7 [-0.9, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( -8611; 3.91 GPU-seconds) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.9)+think@2 acc 0.960 equiv 34910 (+20133; 22.14 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.9)+think@3 acc 0.954 equiv 18892 ( +4115; 11.98 GPU-seconds) | vs agree2 +1.4 [+1.1, +1.7] + AC(0.9)+think@4 acc 0.954 equiv 19471 ( +4695; 12.35 GPU-seconds) | vs agree2 +1.4 [+1.1, +1.6] + AC(0.95) acc 0.876 equiv 7112 ( -7665; 4.51 GPU-seconds) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.95)+think@2 acc 0.960 equiv 34910 (+20133; 22.14 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@3 acc 0.960 equiv 36059 (+21283; 22.87 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@4 acc 0.957 equiv 21701 ( +6924; 13.76 GPU-seconds) | vs agree2 +1.7 [+1.3, +2.1] + AC(0.99) acc 0.876 equiv 8342 ( -6435; 5.29 GPU-seconds) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.99)+think@2 acc 0.960 equiv 34910 (+20133; 22.14 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@3 acc 0.960 equiv 36059 (+21283; 22.87 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@4 acc 0.960 equiv 37209 (+22433; 23.59 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] + always think acc 0.960 equiv 32611 (+17834; 20.68 GPU-seconds) | vs agree2 +2.0 [+1.3, +2.7] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.808 equiv 1396 +R 24000: acc 0.810 equiv 1472 +R 32000: acc 0.829 equiv 2380 +R 48000: acc 0.863 equiv 3899 +R 64000: acc 0.894 equiv 7482 +R 96000: acc 0.924 equiv 11235 +parity: adaptive-SC Bayes at agree2's cost acc 0.945 equiv 14828 (+52) | vs agree2 +0.4 [+0.0, +0.8] | R [209676, 368277] | {'stop@3': 4195, 'stop@2': 1991, 'stop@5': 1092, 'stop@4': 950, 'think:q32b_think@8': 624, 'stop@6': 376, 'think:q32b_think@5': 366, 'stop@7': 263} +split 1: adaptive-SC Bayes at agree2's cost acc 0.944 equiv 14564 (-213) | vs agree2 +0.4 [-0.0, +0.8] | R [267815, 295920] | {'stop@2': 3447, 'stop@3': 2091, 'stop@4': 1802, 'stop@5': 910, 'think:q32b_think@8': 520, 'think:q32b_think@4': 396, 'stop@6': 334, 'think:q32b_think@5': 304} +split 2: adaptive-SC Bayes at agree2's cost acc 0.943 equiv 14373 (-403) | vs agree2 +0.2 [-0.2, +0.6] | R [250752, 288676] | {'stop@2': 2997, 'stop@3': 2894, 'stop@4': 1435, 'stop@5': 1013, 'think:q32b_think@8': 447, 'think:q32b_think@4': 397, 'think:q32b_think@6': 391, 'stop@6': 331} +split 3: adaptive-SC Bayes at agree2's cost acc 0.943 equiv 14790 (+13) | vs agree2 +0.3 [-0.2, +0.8] | R [297679, 308944] | {'stop@2': 3282, 'stop@3': 2624, 'stop@4': 1575, 'stop@5': 756, 'think:q32b_think@8': 661, 'stop@6': 403, 'think:q32b_think@5': 359, 'think:q32b_think@4': 290} +parity at agree2's accuracy: acc 0.936 cost 8.349 GPU-seconds vs 9.37 | cost -10.9% [-14.3, -7.4] | acc -0.5 [-1.0, -0.0] +split 1 at agree2's accuracy: acc 0.938 cost 8.154 GPU-seconds vs 9.37 | cost -13.0% [-16.6, -9.5] | acc -0.3 [-0.8, +0.2] +split 2 at agree2's accuracy: acc 0.936 cost 8.123 GPU-seconds vs 9.37 | cost -13.3% [-16.8, -9.7] | acc -0.4 [-0.9, +0.1] +split 3 at agree2's accuracy: acc 0.935 cost 8.172 GPU-seconds vs 9.37 | cost -12.8% [-16.5, -8.9] | acc -0.6 [-1.2, -0.1] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7bd8ece87bf5ce4fa2747a1c12506e6d4e82fb42 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_gpu_s.provenance.json @@ -0,0 +1,90 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "gpu_s", + "--tier", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/sc32_gpu_s.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:13:20+04:00", + "wall_secs": 66.8, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..6121af77dea027676c7b6e2e10ceee52c208e6f6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa60b51d4c54c38e2706cf4416427395a1a13fd4a199ed9f8b235e4042be2527 +size 183873 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.json b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..052a0d600c3c13cb30f153a2f8a67aedbaa62695 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.json @@ -0,0 +1,500 @@ +{ + "ref": { + "acc": 0.9405767817781043, + "equiv": 4283.8179463629685 + }, + "cost_model": { + "currency": "tokens", + "unit": "base tokens", + "per_equiv": 1.0, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.9405767817781043, + "equiv": 4283.8179463629685, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1124.9408523144746, + "delta_vs_agree2": [ + -0.14979794268919913, + -0.16256429096252756, + -0.1369374540778839 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2249.881704628949, + "delta_vs_agree2": [ + -0.12876561351947097, + -0.14226671565025717, + -0.11544819985304923 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3374.822556943424, + "delta_vs_agree2": [ + -0.09616091109478325, + -0.10911324393828067, + -0.08357825128581925 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4499.763409257898, + "delta_vs_agree2": [ + -0.0852314474650992, + -0.09809652828802351, + -0.07274063188831742 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5624.704261572373, + "delta_vs_agree2": [ + -0.0775165319617928, + -0.09056070903747244, + -0.06493387215282881 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6749.645113886848, + "delta_vs_agree2": [ + -0.07127112417340191, + -0.08440714548126378, + -0.05905584129316679 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 7874.585966201323, + "delta_vs_agree2": [ + -0.068791329904482, + -0.08174136664217486, + -0.05648190668626012 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 8999.526818515797, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4328.9513225569435, + "delta_vs_agree2": [ + -0.06915870683321088, + -0.08238886847905952, + -0.05639235855988244 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.9405767817781043, + "equiv": 4283.8179463629685, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9403930933137399, + "equiv": 4758.662803085966, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9340558412931668, + "equiv": 4695.218185157972, + "delta_vs_agree2": [ + -0.006520940484937546, + -0.009184423218221896, + -0.004132990448199853 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6046.053545187362, + "delta_vs_agree2": [ + -0.06484202792064658, + -0.07834542615723733, + -0.05198383541513593 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9603232916972814, + "equiv": 7567.180749448935, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9542615723732549, + "equiv": 5892.238997060985, + "delta_vs_agree2": [ + 0.013684790595150624, + 0.010929463629684055, + 0.016623806024981633 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9540778839088905, + "equiv": 6471.090558412931, + "delta_vs_agree2": [ + 0.013501102130786187, + 0.010745775165319618, + 0.0162587252020573 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 6970.480804555474, + "delta_vs_agree2": [ + -0.06456649522409992, + -0.07806989346069067, + -0.05152461425422483 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9603232916972814, + "equiv": 7567.180749448935, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9603232916972814, + "equiv": 8692.121601763409, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9572005878030859, + "equiv": 7287.702479794269, + "delta_vs_agree2": [ + 0.016623806024981633, + 0.012950036737692874, + 0.020575404114621592 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8168.442964731815, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9603232916972814, + "equiv": 7567.180749448935, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9603232916972814, + "equiv": 8692.121601763409, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9603232916972814, + "equiv": 9817.062454077884, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "always think": { + "acc": 0.9603232916972814, + "equiv": 5317.299044819985, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7909625275532697, + "equiv": 1126.719911829537 + }, + { + "R": 8000, + "acc": 0.7910543717854519, + "equiv": 1127.2466936076414 + }, + { + "R": 16000, + "acc": 0.93708670095518, + "equiv": 3569.6509919177074 + }, + { + "R": 24000, + "acc": 0.9426891991182954, + "equiv": 3761.4378398236595 + }, + { + "R": 32000, + "acc": 0.9488427626745041, + "equiv": 3929.1393093313736 + }, + { + "R": 48000, + "acc": 0.9541697281410727, + "equiv": 4187.878085966201 + }, + { + "R": 64000, + "acc": 0.9560066127847171, + "equiv": 4303.237656135195 + }, + { + "R": 96000, + "acc": 0.9575679647318148, + "equiv": 4473.639070536371 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9546289493019838, + "equiv": 4215.773769287289, + "R": [ + 91824.58538325278, + 40573.87526947939 + ], + "delta_vs_agree2": [ + 0.0140521675238795, + 0.008541513592946363, + 0.019930198383541514 + ], + "actions": { + "stop@2": 4261, + "stop@5": 253, + "think:q32b_think@2": 144, + "stop@3": 144, + "think:q32b_think@0": 3576, + "stop@6": 40, + "think:q32b_think@8": 17, + "think:q32b_think@7": 20, + "think:q32b_think@5": 9, + "stop@8": 28, + "stop@7": 48, + "think:q32b_think@3": 46, + "think:q32b_think@6": 25, + "think:q32b_think@4": 21, + "think:q32b_think@1": 522, + "stop@1": 1734 + } + }, + { + "seed": 1, + "acc": 0.9538941954445261, + "equiv": 4314.635947832477, + "R": [ + 67328.52078761288, + 55381.77637001384 + ], + "delta_vs_agree2": [ + 0.01331741366642175, + 0.007896307861866275, + 0.019103600293901544 + ], + "actions": { + "stop@8": 62, + "think:q32b_think@3": 104, + "think:q32b_think@2": 318, + "stop@3": 583, + "stop@2": 5062, + "think:q32b_think@0": 3688, + "stop@5": 245, + "stop@7": 57, + "stop@4": 39, + "think:q32b_think@4": 26, + "stop@6": 53, + "think:q32b_think@7": 38, + "think:q32b_think@8": 31, + "think:q32b_think@5": 6, + "think:q32b_think@6": 15, + "stop@1": 561 + } + }, + { + "seed": 2, + "acc": 0.9540778839088905, + "equiv": 4266.510562086701, + "R": [ + 79283.1784206788, + 44078.78098259121 + ], + "delta_vs_agree2": [ + 0.013501102130786187, + 0.008082292432035268, + 0.019289584864070525 + ], + "actions": { + "think:q32b_think@2": 388, + "think:q32b_think@3": 227, + "stop@7": 28, + "stop@6": 39, + "stop@5": 141, + "stop@4": 76, + "stop@3": 173, + "stop@2": 5479, + "think:q32b_think@4": 24, + "think:q32b_think@0": 3600, + "think:q32b_think@7": 4, + "think:q32b_think@8": 7, + "stop@8": 57, + "think:q32b_think@5": 10, + "think:q32b_think@6": 14, + "stop@1": 621 + } + }, + { + "seed": 3, + "acc": 0.9544452608376194, + "equiv": 4242.00922116091, + "R": [ + 49402.412840278674, + 66141.97703218769 + ], + "delta_vs_agree2": [ + 0.013868479059515063, + 0.008541513592946363, + 0.019562821454812637 + ], + "actions": { + "stop@2": 5320, + "think:q32b_think@1": 399, + "stop@1": 616, + "stop@5": 148, + "think:q32b_think@0": 3648, + "think:q32b_think@3": 53, + "stop@6": 21, + "think:q32b_think@4": 26, + "stop@7": 34, + "think:q32b_think@8": 16, + "think:q32b_think@2": 335, + "think:q32b_think@5": 4, + "stop@8": 21, + "stop@4": 29, + "think:q32b_think@6": 3, + "think:q32b_think@7": 6, + "stop@3": 209 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9426891991182954, + "equiv": 3761.095481263777, + "R": [ + 22358.663294702732, + 22482.003544281713 + ], + "delta_acc": [ + 0.002112417340191036, + -0.00395159808963997, + 0.009000734753857457 + ], + "rel_cost": [ + -0.12202256763572124, + -0.14920852463555445, + -0.09306146425335104 + ] + }, + { + "seed": 1, + "acc": 0.942138133725202, + "equiv": 3769.3541513592945, + "R": [ + 23367.790099632886, + 27279.94519816034 + ], + "delta_acc": [ + 0.0015613519470977222, + -0.004410819250551065, + 0.00789860396767083 + ], + "rel_cost": [ + -0.12009469156840848, + -0.14815632663270598, + -0.09143833511078628 + ] + }, + { + "seed": 2, + "acc": 0.9412196914033799, + "equiv": 3721.1166605437184, + "R": [ + 20256.816931905476, + 21697.90831770529 + ], + "delta_acc": [ + 0.0006429096252755327, + -0.005418809698750918, + 0.007531227038941954 + ], + "rel_cost": [ + -0.13135508858330291, + -0.16029160822478866, + -0.10237103313659894 + ] + }, + { + "seed": 3, + "acc": 0.9424136664217487, + "equiv": 3732.863409257898, + "R": [ + 26373.01397547303, + 22770.81386127522 + ], + "delta_acc": [ + 0.0018368846436443791, + -0.004227130786186627, + 0.008451965466568687 + ], + "rel_cost": [ + -0.12861296721837565, + -0.15635279023593773, + -0.10022076741312545 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.log b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.log new file mode 100644 index 0000000000000000000000000000000000000000..6524239c495a4ccad548cbe7aeaeba2c5a709bf5 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,329 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> q32b_think): acc 0.941 equiv 4284 +cost model tokens: 1 equiv = 1 base tokens; agree2 costs 4284 base tokens per problem + agree2 acc 0.941 equiv 4284 ( +0; 4284 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1125 ( -3159; 1125 base tokens) | vs agree2 -15.0 [-16.3, -13.7] + SC@2 acc 0.812 equiv 2250 ( -2034; 2250 base tokens) | vs agree2 -12.9 [-14.2, -11.5] + SC@3 acc 0.844 equiv 3375 ( -909; 3375 base tokens) | vs agree2 -9.6 [-10.9, -8.4] + SC@4 acc 0.855 equiv 4500 ( +216; 4500 base tokens) | vs agree2 -8.5 [-9.8, -7.3] + SC@5 acc 0.863 equiv 5625 ( +1341; 5625 base tokens) | vs agree2 -7.8 [-9.1, -6.5] + SC@6 acc 0.869 equiv 6750 ( +2466; 6750 base tokens) | vs agree2 -7.1 [-8.4, -5.9] + SC@7 acc 0.872 equiv 7875 ( +3591; 7875 base tokens) | vs agree2 -6.9 [-8.2, -5.6] + SC@8 acc 0.876 equiv 9000 ( +4716; 9000 base tokens) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.8) acc 0.871 equiv 4329 ( +45; 4329 base tokens) | vs agree2 -6.9 [-8.2, -5.6] + AC(0.8)+think@2 acc 0.941 equiv 4284 ( +0; 4284 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.940 equiv 4759 ( +475; 4759 base tokens) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.934 equiv 4695 ( +411; 4695 base tokens) | vs agree2 -0.7 [-0.9, -0.4] + AC(0.9) acc 0.876 equiv 6046 ( +1762; 6046 base tokens) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.9)+think@2 acc 0.960 equiv 7567 ( +3283; 7567 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.9)+think@3 acc 0.954 equiv 5892 ( +1608; 5892 base tokens) | vs agree2 +1.4 [+1.1, +1.7] + AC(0.9)+think@4 acc 0.954 equiv 6471 ( +2187; 6471 base tokens) | vs agree2 +1.4 [+1.1, +1.6] + AC(0.95) acc 0.876 equiv 6970 ( +2687; 6970 base tokens) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.95)+think@2 acc 0.960 equiv 7567 ( +3283; 7567 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@3 acc 0.960 equiv 8692 ( +4408; 8692 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@4 acc 0.957 equiv 7288 ( +3004; 7288 base tokens) | vs agree2 +1.7 [+1.3, +2.1] + AC(0.99) acc 0.876 equiv 8168 ( +3885; 8168 base tokens) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.99)+think@2 acc 0.960 equiv 7567 ( +3283; 7567 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@3 acc 0.960 equiv 8692 ( +4408; 8692 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@4 acc 0.960 equiv 9817 ( +5533; 9817 base tokens) | vs agree2 +2.0 [+1.3, +2.7] + always think acc 0.960 equiv 5317 ( +1033; 5317 base tokens) | vs agree2 +2.0 [+1.3, +2.7] +R 4000: acc 0.791 equiv 1127 +R 8000: acc 0.791 equiv 1127 +R 16000: acc 0.937 equiv 3570 +R 24000: acc 0.943 equiv 3761 +R 32000: acc 0.949 equiv 3929 +R 48000: acc 0.954 equiv 4188 +R 64000: acc 0.956 equiv 4303 +R 96000: acc 0.958 equiv 4474 +parity: adaptive-SC Bayes at agree2's cost acc 0.955 equiv 4216 (-68) | vs agree2 +1.4 [+0.9, +2.0] | R [91825, 40574] | {'stop@2': 4261, 'think:q32b_think@0': 3576, 'stop@1': 1734, 'think:q32b_think@1': 522, 'stop@5': 253, 'think:q32b_think@2': 144, 'stop@3': 144, 'stop@7': 48} +split 1: adaptive-SC Bayes at agree2's cost acc 0.954 equiv 4315 (+31) | vs agree2 +1.3 [+0.8, +1.9] | R [67329, 55382] | {'stop@2': 5062, 'think:q32b_think@0': 3688, 'stop@3': 583, 'stop@1': 561, 'think:q32b_think@2': 318, 'stop@5': 245, 'think:q32b_think@3': 104, 'stop@8': 62} +split 2: adaptive-SC Bayes at agree2's cost acc 0.954 equiv 4267 (-17) | vs agree2 +1.4 [+0.8, +1.9] | R [79283, 44079] | {'stop@2': 5479, 'think:q32b_think@0': 3600, 'stop@1': 621, 'think:q32b_think@2': 388, 'think:q32b_think@3': 227, 'stop@3': 173, 'stop@5': 141, 'stop@4': 76} +split 3: adaptive-SC Bayes at agree2's cost acc 0.954 equiv 4242 (-42) | vs agree2 +1.4 [+0.9, +2.0] | R [49402, 66142] | {'stop@2': 5320, 'think:q32b_think@0': 3648, 'stop@1': 616, 'think:q32b_think@1': 399, 'think:q32b_think@2': 335, 'stop@3': 209, 'stop@5': 148, 'think:q32b_think@3': 53} +parity at agree2's accuracy: acc 0.943 cost 3761 base tokens vs 4284 | cost -12.2% [-14.9, -9.3] | acc +0.2 [-0.4, +0.9] +split 1 at agree2's accuracy: acc 0.942 cost 3769 base tokens vs 4284 | cost -12.0% [-14.8, -9.1] | acc +0.2 [-0.4, +0.8] +split 2 at agree2's accuracy: acc 0.941 cost 3721 base tokens vs 4284 | cost -13.1% [-16.0, -10.2] | acc +0.1 [-0.5, +0.8] +split 3 at agree2's accuracy: acc 0.942 cost 3733 base tokens vs 4284 | cost -12.9% [-15.6, -10.0] | acc +0.2 [-0.4, +0.8] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..43898798f38fd927ff91d6bb7bb0cd1707d5d327 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_tokens.provenance.json @@ -0,0 +1,90 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "tokens", + "--tier", + "q32b_think=workdir/bayes/q32b_think_tier.json", + "--match-acc", + "--out", + "workdir/bayes/v8/sc32_tokens.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:56+04:00", + "wall_secs": 43.7, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..172f639bf6343e69b9e2092f4fbdf8832fdb0310 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba1d74f9e342a7f1d4e7eb678bdc03ee287fddf2c81be01eb242d1e834f7860 +size 189757 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.json new file mode 100644 index 0000000000000000000000000000000000000000..a1d8618f4a00e5f6447c30898d6140a23270b77a --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.json @@ -0,0 +1,499 @@ +{ + "ref": { + "acc": 0.9405767817781043, + "equiv": 5150.545609845701 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "cheapest", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.9405767817781043, + "equiv": 5150.545609845701, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.14979794268919913, + -0.16256429096252756, + -0.1369374540778839 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.12876561351947097, + -0.14226671565025717, + -0.11544819985304923 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.09616091109478325, + -0.10911324393828067, + -0.08357825128581925 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.0852314474650992, + -0.09809652828802351, + -0.07274063188831742 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0775165319617928, + -0.09056070903747244, + -0.06493387215282881 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.07127112417340191, + -0.08440714548126378, + -0.05905584129316679 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.068791329904482, + -0.08174136664217486, + -0.05648190668626012 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.06915870683321088, + -0.08238886847905952, + -0.05639235855988244 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.9405767817781043, + "equiv": 5150.545609845701, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9403930933137399, + "equiv": 5632.626047024248, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9340558412931668, + "equiv": 5360.495435341661, + "delta_vs_agree2": [ + -0.006520940484937546, + -0.009184423218221896, + -0.004132990448199853 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.06484202792064658, + -0.07834542615723733, + -0.05198383541513593 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9603232916972814, + "equiv": 9755.271271124173, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9542615723732549, + "equiv": 6977.969140337987, + "delta_vs_agree2": [ + 0.013684790595150624, + 0.010929463629684055, + 0.016623806024981633 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9540778839088905, + "equiv": 7566.024435157972, + "delta_vs_agree2": [ + 0.013501102130786187, + 0.010745775165319618, + 0.0162587252020573 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.06456649522409992, + -0.07806989346069067, + -0.05152461425422483 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9603232916972814, + "equiv": 9755.271271124173, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9603232916972814, + "equiv": 10904.877810433503, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9572005878030859, + "equiv": 8506.850459221161, + "delta_vs_agree2": [ + 0.016623806024981633, + 0.012950036737692874, + 0.020575404114621592 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9603232916972814, + "equiv": 9755.271271124173, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9603232916972814, + "equiv": 10904.877810433503, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9603232916972814, + "equiv": 12054.484349742836, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "always think": { + "acc": 0.9603232916972814, + "equiv": 7456.05819250551, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 16000, + "acc": 0.8692138133725202, + "equiv": 3077.968547942689 + }, + { + "R": 24000, + "acc": 0.9411278471711977, + "equiv": 4937.058826230713 + }, + { + "R": 32000, + "acc": 0.9473732549595886, + "equiv": 5096.705244305658 + }, + { + "R": 48000, + "acc": 0.9528839088905217, + "equiv": 5359.611760653932 + }, + { + "R": 64000, + "acc": 0.955088170462895, + "equiv": 5517.903719691403 + }, + { + "R": 96000, + "acc": 0.9560984570168993, + "equiv": 5646.251644011756 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9493019838354151, + "equiv": 5179.889915503306, + "R": [ + 44001.89742887743, + 30172.35271204413 + ], + "delta_vs_agree2": [ + 0.008725202057310802, + 0.0029390154298310064, + 0.014972905951506233 + ], + "actions": { + "stop@1": 3414, + "stop@2": 2949, + "stop@4": 139, + "think:q32b_think@2": 271, + "think:q32b_think@0": 3576, + "stop@5": 160, + "stop@6": 39, + "stop@7": 65, + "think:q32b_think@7": 19, + "think:q32b_think@5": 12, + "think:q32b_think@3": 15, + "stop@8": 27, + "think:q32b_think@4": 31, + "think:q32b_think@6": 7, + "think:q32b_think@8": 3, + "stop@3": 161 + } + }, + { + "seed": 1, + "acc": 0.9442505510653931, + "equiv": 5077.934055841293, + "R": [ + 36742.20417604087, + 30332.785879190633 + ], + "delta_vs_agree2": [ + 0.0036737692872887582, + -0.002387950036737693, + 0.010105161645848628 + ], + "actions": { + "stop@2": 2279, + "think:q32b_think@3": 82, + "stop@4": 205, + "think:q32b_think@0": 3688, + "stop@1": 4215, + "stop@7": 73, + "think:q32b_think@2": 104, + "stop@6": 34, + "stop@3": 49, + "think:q32b_think@7": 11, + "think:q32b_think@5": 20, + "stop@5": 60, + "think:q32b_think@8": 14, + "think:q32b_think@4": 15, + "stop@8": 37, + "think:q32b_think@6": 2 + } + }, + { + "seed": 2, + "acc": 0.946730345334313, + "equiv": 5108.292932586333, + "R": [ + 43714.70455047698, + 32839.90794532481 + ], + "delta_vs_agree2": [ + 0.00615356355620867, + 0.0002755326965466569, + 0.012398971344599559 + ], + "actions": { + "stop@5": 206, + "stop@6": 31, + "stop@2": 2611, + "stop@1": 3926, + "think:q32b_think@3": 73, + "think:q32b_think@2": 152, + "think:q32b_think@4": 18, + "think:q32b_think@0": 3600, + "stop@3": 119, + "think:q32b_think@8": 14, + "think:q32b_think@7": 11, + "stop@7": 46, + "think:q32b_think@5": 44, + "stop@8": 27, + "stop@4": 9, + "think:q32b_think@6": 1 + } + }, + { + "seed": 3, + "acc": 0.9483835415135929, + "equiv": 5101.276542983101, + "R": [ + 39017.86102873052, + 32700.663999164928 + ], + "delta_vs_agree2": [ + 0.007806759735488611, + 0.002204261572373255, + 0.013868479059515063 + ], + "actions": { + "stop@1": 3963, + "stop@2": 2498, + "think:q32b_think@2": 258, + "stop@5": 99, + "stop@6": 22, + "think:q32b_think@0": 3648, + "stop@4": 169, + "stop@3": 77, + "stop@8": 24, + "stop@7": 40, + "think:q32b_think@5": 14, + "think:q32b_think@4": 14, + "think:q32b_think@8": 6, + "think:q32b_think@6": 5, + "think:q32b_think@7": 3, + "think:q32b_think@3": 48 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9410360029390155, + "equiv": 4936.350133174136, + "R": [ + 23403.76818773052, + 23382.30415765745 + ], + "delta_acc": [ + 0.0004592211609110948, + -0.005696638501102131, + 0.007439382806759736 + ], + "rel_cost": [ + -0.041586948819968196, + -0.07185071633836701, + -0.00932253431323091 + ] + }, + { + "seed": 1, + "acc": 0.9412196914033799, + "equiv": 4956.526010286553, + "R": [ + 24460.01596270518, + 28000.174242016714 + ], + "delta_acc": [ + 0.0006429096252755327, + -0.005602498163115356, + 0.007074301983835403 + ], + "rel_cost": [ + -0.03766971778451256, + -0.0694605956345558, + -0.0055554942161507 + ] + }, + { + "seed": 2, + "acc": 0.9386480529022777, + "equiv": 4866.483096069066, + "R": [ + 23805.320187558315, + 22665.287727281033 + ], + "delta_acc": [ + -0.001928728875826598, + -0.008451965466568699, + 0.004961884643644367 + ], + "rel_cost": [ + -0.055151926668434006, + -0.08675321443852511, + -0.02268254455026369 + ] + }, + { + "seed": 3, + "acc": 0.9412196914033799, + "equiv": 4917.07822373255, + "R": [ + 26944.758055139475, + 24528.504861426587 + ], + "delta_acc": [ + 0.0006429096252755327, + -0.005602498163115356, + 0.007439382806759736 + ], + "rel_cost": [ + -0.04532867074642724, + -0.0759534618073302, + -0.013568378020935667 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.log b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.log new file mode 100644 index 0000000000000000000000000000000000000000..5366168fc3826d41b25699133e81ddb457c1279d --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,489 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> q32b_think): acc 0.941 equiv 5151 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.00103 USD per problem + agree2 acc 0.941 equiv 5151 ( +0; 0.00103 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -4001; 0.0002299 USD) | vs agree2 -15.0 [-16.3, -13.7] + SC@2 acc 0.812 equiv 2299 ( -2851; 0.0004598 USD) | vs agree2 -12.9 [-14.2, -11.5] + SC@3 acc 0.844 equiv 3449 ( -1702; 0.0006898 USD) | vs agree2 -9.6 [-10.9, -8.4] + SC@4 acc 0.855 equiv 4598 ( -552; 0.0009197 USD) | vs agree2 -8.5 [-9.8, -7.3] + SC@5 acc 0.863 equiv 5748 ( +597; 0.00115 USD) | vs agree2 -7.8 [-9.1, -6.5] + SC@6 acc 0.869 equiv 6898 ( +1747; 0.00138 USD) | vs agree2 -7.1 [-8.4, -5.9] + SC@7 acc 0.872 equiv 8047 ( +2897; 0.001609 USD) | vs agree2 -6.9 [-8.2, -5.6] + SC@8 acc 0.876 equiv 9197 ( +4046; 0.001839 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.8) acc 0.871 equiv 4412 ( -739; 0.0008824 USD) | vs agree2 -6.9 [-8.2, -5.6] + AC(0.8)+think@2 acc 0.941 equiv 5151 ( +0; 0.00103 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.940 equiv 5633 ( +482; 0.001127 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.934 equiv 5360 ( +210; 0.001072 USD) | vs agree2 -0.7 [-0.9, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1015; 0.001233 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.9)+think@2 acc 0.960 equiv 9755 ( +4605; 0.001951 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.9)+think@3 acc 0.954 equiv 6978 ( +1827; 0.001396 USD) | vs agree2 +1.4 [+1.1, +1.7] + AC(0.9)+think@4 acc 0.954 equiv 7566 ( +2415; 0.001513 USD) | vs agree2 +1.4 [+1.1, +1.6] + AC(0.95) acc 0.876 equiv 7111 ( +1961; 0.001422 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.95)+think@2 acc 0.960 equiv 9755 ( +4605; 0.001951 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@3 acc 0.960 equiv 10905 ( +5754; 0.002181 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@4 acc 0.957 equiv 8507 ( +3356; 0.001701 USD) | vs agree2 +1.7 [+1.3, +2.1] + AC(0.99) acc 0.876 equiv 8341 ( +3191; 0.001668 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.99)+think@2 acc 0.960 equiv 9755 ( +4605; 0.001951 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@3 acc 0.960 equiv 10905 ( +5754; 0.002181 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@4 acc 0.960 equiv 12054 ( +6904; 0.002411 USD) | vs agree2 +2.0 [+1.3, +2.7] + always think acc 0.960 equiv 7456 ( +2306; 0.001491 USD) | vs agree2 +2.0 [+1.3, +2.7] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.869 equiv 3078 +R 24000: acc 0.941 equiv 4937 +R 32000: acc 0.947 equiv 5097 +R 48000: acc 0.953 equiv 5360 +R 64000: acc 0.955 equiv 5518 +R 96000: acc 0.956 equiv 5646 +parity: adaptive-SC Bayes at agree2's cost acc 0.949 equiv 5180 (+29) | vs agree2 +0.9 [+0.3, +1.5] | R [44002, 30172] | {'think:q32b_think@0': 3576, 'stop@1': 3414, 'stop@2': 2949, 'think:q32b_think@2': 271, 'stop@3': 161, 'stop@5': 160, 'stop@4': 139, 'stop@7': 65} +split 1: adaptive-SC Bayes at agree2's cost acc 0.944 equiv 5078 (-73) | vs agree2 +0.4 [-0.2, +1.0] | R [36742, 30333] | {'stop@1': 4215, 'think:q32b_think@0': 3688, 'stop@2': 2279, 'stop@4': 205, 'think:q32b_think@2': 104, 'think:q32b_think@3': 82, 'stop@7': 73, 'stop@5': 60} +split 2: adaptive-SC Bayes at agree2's cost acc 0.947 equiv 5108 (-42) | vs agree2 +0.6 [+0.0, +1.2] | R [43715, 32840] | {'stop@1': 3926, 'think:q32b_think@0': 3600, 'stop@2': 2611, 'stop@5': 206, 'think:q32b_think@2': 152, 'stop@3': 119, 'think:q32b_think@3': 73, 'stop@7': 46} +split 3: adaptive-SC Bayes at agree2's cost acc 0.948 equiv 5101 (-49) | vs agree2 +0.8 [+0.2, +1.4] | R [39018, 32701] | {'stop@1': 3963, 'think:q32b_think@0': 3648, 'stop@2': 2498, 'think:q32b_think@2': 258, 'stop@4': 169, 'stop@5': 99, 'stop@3': 77, 'think:q32b_think@3': 48} +parity at agree2's accuracy: acc 0.941 cost 0.0009873 USD vs 0.00103 | cost -4.2% [-7.2, -0.9] | acc +0.0 [-0.6, +0.7] +split 1 at agree2's accuracy: acc 0.941 cost 0.0009913 USD vs 0.00103 | cost -3.8% [-6.9, -0.6] | acc +0.1 [-0.6, +0.7] +split 2 at agree2's accuracy: acc 0.939 cost 0.0009733 USD vs 0.00103 | cost -5.5% [-8.7, -2.3] | acc -0.2 [-0.8, +0.5] +split 3 at agree2's accuracy: acc 0.941 cost 0.0009834 USD vs 0.00103 | cost -4.5% [-7.6, -1.4] | acc +0.1 [-0.6, +0.7] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d2e212e480d7eb5858b96f10938e66d1b88cf960 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_cheapest.provenance.json @@ -0,0 +1,92 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "cheapest", + "--tier", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/sc32_usd_cheapest.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:56+04:00", + "wall_secs": 43.6, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..de15b0bb0c4ff98c431a7389693972aba7022d83 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5095d5f10cc996fa92e5a2ab199e4a7618adb98f77e7d2b452c5a1e4b5fed127 +size 194848 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd62e4435dc7890a0961fb222057383da6eabf3 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.json @@ -0,0 +1,494 @@ +{ + "ref": { + "acc": 0.9405767817781043, + "equiv": 8100.071643093312 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "dear_32b", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.9405767817781043, + "equiv": 8100.071643093312, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.14979794268919913, + -0.16256429096252756, + -0.1369374540778839 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.12876561351947097, + -0.14226671565025717, + -0.11544819985304923 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.09616091109478325, + -0.10911324393828067, + -0.08357825128581925 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.0852314474650992, + -0.09809652828802351, + -0.07274063188831742 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0775165319617928, + -0.09056070903747244, + -0.06493387215282881 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.07127112417340191, + -0.08440714548126378, + -0.05905584129316679 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.068791329904482, + -0.08174136664217486, + -0.05648190668626012 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.06915870683321088, + -0.08238886847905952, + -0.05639235855988244 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.9405767817781043, + "equiv": 8100.071643093312, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9403930933137399, + "equiv": 8579.525822005875, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9340558412931668, + "equiv": 7525.915664033799, + "delta_vs_agree2": [ + -0.006520940484937546, + -0.009184423218221896, + -0.004132990448199853 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.06484202792064658, + -0.07834542615723733, + -0.05198383541513593 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9603232916972814, + "equiv": 17466.341513592943, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9542615723732549, + "equiv": 10628.620187362234, + "delta_vs_agree2": [ + 0.013684790595150624, + 0.010929463629684055, + 0.016623806024981633 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9540778839088905, + "equiv": 11214.049223916236, + "delta_vs_agree2": [ + 0.013501102130786187, + 0.010745775165319618, + 0.0162587252020573 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.06456649522409992, + -0.07806989346069067, + -0.05152461425422483 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9603232916972814, + "equiv": 17466.341513592943, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9603232916972814, + "equiv": 18615.948052902277, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9572005878030859, + "equiv": 12549.804578434974, + "delta_vs_agree2": [ + 0.016623806024981633, + 0.012950036737692874, + 0.020575404114621592 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9603232916972814, + "equiv": 17466.341513592943, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9603232916972814, + "equiv": 18615.948052902277, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9603232916972814, + "equiv": 19765.554592211607, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "always think": { + "acc": 0.9603232916972814, + "equiv": 15167.128434974282, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 16000, + "acc": 0.8075863335782513, + "equiv": 1396.3716017634092 + }, + { + "R": 24000, + "acc": 0.8149338721528288, + "equiv": 1676.2500183688464 + }, + { + "R": 32000, + "acc": 0.901451138868479, + "equiv": 5596.907228141072 + }, + { + "R": 48000, + "acc": 0.9304739162380602, + "equiv": 7248.4472033431275 + }, + { + "R": 64000, + "acc": 0.9386480529022777, + "equiv": 7845.916568699485 + }, + { + "R": 96000, + "acc": 0.9466385011021308, + "equiv": 9170.761462160175 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9389235855988244, + "equiv": 7960.538997060984, + "R": [ + 65162.81182746214, + 87240.34686240471 + ], + "delta_vs_agree2": [ + -0.0016531961792799412, + -0.005602498163115356, + 0.002112417340191036 + ], + "actions": { + "stop@2": 7205, + "stop@4": 420, + "stop@5": 435, + "stop@6": 130, + "stop@7": 134, + "think:q32b_think@6": 95, + "stop@3": 613, + "think:q32b_think@2": 633, + "think:q32b_think@3": 288, + "think:q32b_think@4": 489, + "stop@8": 172, + "think:q32b_think@7": 30, + "think:q32b_think@5": 196, + "think:q32b_think@1": 1, + "think:q32b_think@8": 47 + } + }, + { + "seed": 1, + "acc": 0.9402094048493754, + "equiv": 7959.679114621601, + "R": [ + 66863.9667712638, + 89949.39614832334 + ], + "delta_vs_agree2": [ + -0.0003673769287288758, + -0.004041146216017634, + 0.0034900808229243203 + ], + "actions": { + "stop@8": 132, + "stop@7": 141, + "stop@6": 154, + "stop@4": 273, + "stop@3": 773, + "stop@2": 7096, + "think:q32b_think@3": 459, + "stop@5": 491, + "think:q32b_think@4": 225, + "think:q32b_think@2": 817, + "think:q32b_think@7": 67, + "think:q32b_think@6": 133, + "think:q32b_think@5": 89, + "think:q32b_think@8": 37, + "think:q32b_think@1": 1 + } + }, + { + "seed": 2, + "acc": 0.9414952240999265, + "equiv": 8091.537027002203, + "R": [ + 70038.00058507305, + 76742.53748397072 + ], + "delta_vs_agree2": [ + 0.0009184423218221896, + -0.0030308596620132254, + 0.004775900073475386 + ], + "actions": { + "stop@5": 587, + "stop@8": 164, + "stop@7": 117, + "stop@6": 145, + "stop@4": 390, + "stop@3": 508, + "stop@2": 7112, + "think:q32b_think@6": 71, + "think:q32b_think@2": 611, + "think:q32b_think@3": 686, + "think:q32b_think@4": 177, + "think:q32b_think@5": 205, + "think:q32b_think@8": 76, + "think:q32b_think@7": 39 + } + }, + { + "seed": 3, + "acc": 0.9407604702424688, + "equiv": 8151.956695444526, + "R": [ + 95584.05852332129, + 55620.8972776458 + ], + "delta_vs_agree2": [ + 0.0001836884643644379, + -0.004041146216017634, + 0.004224834680382072 + ], + "actions": { + "stop@2": 6370, + "stop@6": 209, + "think:q32b_think@3": 639, + "stop@5": 710, + "stop@3": 1259, + "stop@4": 231, + "think:q32b_think@6": 37, + "think:q32b_think@5": 183, + "stop@8": 98, + "stop@7": 122, + "think:q32b_think@7": 52, + "think:q32b_think@8": 35, + "think:q32b_think@4": 213, + "think:q32b_think@2": 728, + "think:q32b_think@1": 2 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9443423952975754, + "equiv": 8886.334340558411, + "R": [ + 65163.05893015895, + 59330.632516829646 + ], + "delta_acc": [ + 0.003765613519470977, + -0.0011044268919911829, + 0.008633357825128581 + ], + "rel_cost": [ + 0.09706861026785137, + 0.06353250901962904, + 0.13124616628486666 + ] + }, + { + "seed": 1, + "acc": 0.9376377663482733, + "equiv": 7669.243277002204, + "R": [ + 53842.244973533416, + 58447.950800650295 + ], + "delta_acc": [ + -0.0029390154298310064, + -0.00707200587803086, + 0.0011939750183688465 + ], + "rel_cost": [ + -0.05318821673120184, + -0.07677697942878778, + -0.030730151797483382 + ] + }, + { + "seed": 2, + "acc": 0.9376377663482733, + "equiv": 7766.884827332842, + "R": [ + 57581.5891689974, + 61119.50696680797 + ], + "delta_acc": [ + -0.0029390154298310064, + -0.007255694342395298, + 0.0013776634827332844 + ], + "rel_cost": [ + -0.041133810963828746, + -0.06676285007550273, + -0.014911697390608412 + ] + }, + { + "seed": 3, + "acc": 0.9388317413666422, + "equiv": 7926.542427443055, + "R": [ + 67255.20709236493, + 52691.11102712724 + ], + "delta_acc": [ + -0.0017450404114621601, + -0.006061719324026451, + 0.002296105804555474 + ], + "rel_cost": [ + -0.021423170472599472, + -0.045324401010637935, + 0.0024378930119188617 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.log b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.log new file mode 100644 index 0000000000000000000000000000000000000000..893b768cac9e70edf7dde8e94ac838d86c685e55 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,505 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> q32b_think): acc 0.941 equiv 8100 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.00162 USD per problem + agree2 acc 0.941 equiv 8100 ( +0; 0.00162 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -6950; 0.0002299 USD) | vs agree2 -15.0 [-16.3, -13.7] + SC@2 acc 0.812 equiv 2299 ( -5801; 0.0004598 USD) | vs agree2 -12.9 [-14.2, -11.5] + SC@3 acc 0.844 equiv 3449 ( -4651; 0.0006898 USD) | vs agree2 -9.6 [-10.9, -8.4] + SC@4 acc 0.855 equiv 4598 ( -3502; 0.0009197 USD) | vs agree2 -8.5 [-9.8, -7.3] + SC@5 acc 0.863 equiv 5748 ( -2352; 0.00115 USD) | vs agree2 -7.8 [-9.1, -6.5] + SC@6 acc 0.869 equiv 6898 ( -1202; 0.00138 USD) | vs agree2 -7.1 [-8.4, -5.9] + SC@7 acc 0.872 equiv 8047 ( -53; 0.001609 USD) | vs agree2 -6.9 [-8.2, -5.6] + SC@8 acc 0.876 equiv 9197 ( +1097; 0.001839 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.8) acc 0.871 equiv 4412 ( -3688; 0.0008824 USD) | vs agree2 -6.9 [-8.2, -5.6] + AC(0.8)+think@2 acc 0.941 equiv 8100 ( +0; 0.00162 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.940 equiv 8580 ( +479; 0.001716 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.934 equiv 7526 ( -574; 0.001505 USD) | vs agree2 -0.7 [-0.9, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( -1935; 0.001233 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.9)+think@2 acc 0.960 equiv 17466 ( +9366; 0.003493 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.9)+think@3 acc 0.954 equiv 10629 ( +2529; 0.002126 USD) | vs agree2 +1.4 [+1.1, +1.7] + AC(0.9)+think@4 acc 0.954 equiv 11214 ( +3114; 0.002243 USD) | vs agree2 +1.4 [+1.1, +1.6] + AC(0.95) acc 0.876 equiv 7111 ( -989; 0.001422 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.95)+think@2 acc 0.960 equiv 17466 ( +9366; 0.003493 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@3 acc 0.960 equiv 18616 (+10516; 0.003723 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@4 acc 0.957 equiv 12550 ( +4450; 0.00251 USD) | vs agree2 +1.7 [+1.3, +2.1] + AC(0.99) acc 0.876 equiv 8341 ( +241; 0.001668 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.99)+think@2 acc 0.960 equiv 17466 ( +9366; 0.003493 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@3 acc 0.960 equiv 18616 (+10516; 0.003723 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@4 acc 0.960 equiv 19766 (+11665; 0.003953 USD) | vs agree2 +2.0 [+1.3, +2.7] + always think acc 0.960 equiv 15167 ( +7067; 0.003033 USD) | vs agree2 +2.0 [+1.3, +2.7] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.808 equiv 1396 +R 24000: acc 0.815 equiv 1676 +R 32000: acc 0.901 equiv 5597 +R 48000: acc 0.930 equiv 7248 +R 64000: acc 0.939 equiv 7846 +R 96000: acc 0.947 equiv 9171 +parity: adaptive-SC Bayes at agree2's cost acc 0.939 equiv 7961 (-140) | vs agree2 -0.2 [-0.6, +0.2] | R [65163, 87240] | {'stop@2': 7205, 'think:q32b_think@2': 633, 'stop@3': 613, 'think:q32b_think@4': 489, 'stop@5': 435, 'stop@4': 420, 'think:q32b_think@3': 288, 'think:q32b_think@5': 196} +split 1: adaptive-SC Bayes at agree2's cost acc 0.940 equiv 7960 (-140) | vs agree2 -0.0 [-0.4, +0.3] | R [66864, 89949] | {'stop@2': 7096, 'think:q32b_think@2': 817, 'stop@3': 773, 'stop@5': 491, 'think:q32b_think@3': 459, 'stop@4': 273, 'think:q32b_think@4': 225, 'stop@6': 154} +split 2: adaptive-SC Bayes at agree2's cost acc 0.941 equiv 8092 (-9) | vs agree2 +0.1 [-0.3, +0.5] | R [70038, 76743] | {'stop@2': 7112, 'think:q32b_think@3': 686, 'think:q32b_think@2': 611, 'stop@5': 587, 'stop@3': 508, 'stop@4': 390, 'think:q32b_think@5': 205, 'think:q32b_think@4': 177} +split 3: adaptive-SC Bayes at agree2's cost acc 0.941 equiv 8152 (+52) | vs agree2 +0.0 [-0.4, +0.4] | R [95584, 55621] | {'stop@2': 6370, 'stop@3': 1259, 'think:q32b_think@2': 728, 'stop@5': 710, 'think:q32b_think@3': 639, 'stop@4': 231, 'think:q32b_think@4': 213, 'stop@6': 209} +parity at agree2's accuracy: acc 0.944 cost 0.001777 USD vs 0.00162 | cost +9.7% [+6.4, +13.1] | acc +0.4 [-0.1, +0.9] +split 1 at agree2's accuracy: acc 0.938 cost 0.001534 USD vs 0.00162 | cost -5.3% [-7.7, -3.1] | acc -0.3 [-0.7, +0.1] +split 2 at agree2's accuracy: acc 0.938 cost 0.001553 USD vs 0.00162 | cost -4.1% [-6.7, -1.5] | acc -0.3 [-0.7, +0.1] +split 3 at agree2's accuracy: acc 0.939 cost 0.001585 USD vs 0.00162 | cost -2.1% [-4.5, +0.2] | acc -0.2 [-0.6, +0.2] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..bdde2095a7f0bf0583d200ddc6d53930a7c3bc91 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_dear_32b.provenance.json @@ -0,0 +1,92 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "dear_32b", + "--tier", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/sc32_usd_dear_32b.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:13:08+04:00", + "wall_secs": 55.6, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..82d75e166795c77b426d6589384664b9d956cc4b --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbe6f2b9b039025e4a849c82bbfb3d8c9096cd4cce882bfac7175f654b5258c +size 251070 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.json new file mode 100644 index 0000000000000000000000000000000000000000..ee68c51d578347bdf68f246da53f9b8e8e7db4e6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.json @@ -0,0 +1,499 @@ +{ + "ref": { + "acc": 0.9405767817781043, + "equiv": 3553.955527206078 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "scenario": "openrouter_only", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.9405767817781043, + "equiv": 3553.955527206078, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1150.3112732234702, + "delta_vs_agree2": [ + -0.14979794268919913, + -0.16256429096252756, + -0.1369374540778839 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2300.6225464469403, + "delta_vs_agree2": [ + -0.12876561351947097, + -0.14226671565025717, + -0.11544819985304923 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3450.9338196704107, + "delta_vs_agree2": [ + -0.09616091109478325, + -0.10911324393828067, + -0.08357825128581925 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4601.245092893881, + "delta_vs_agree2": [ + -0.0852314474650992, + -0.09809652828802351, + -0.07274063188831742 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5751.556366117351, + "delta_vs_agree2": [ + -0.0775165319617928, + -0.09056070903747244, + -0.06493387215282881 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6901.867639340821, + "delta_vs_agree2": [ + -0.07127112417340191, + -0.08440714548126378, + -0.05905584129316679 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8052.178912564291, + "delta_vs_agree2": [ + -0.068791329904482, + -0.08174136664217486, + -0.05648190668626012 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9202.490185787761, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4414.206011861026, + "delta_vs_agree2": [ + -0.06915870683321088, + -0.08238886847905952, + -0.05639235855988244 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.9405767817781043, + "equiv": 3553.955527206078, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9403930933137399, + "equiv": 4037.685613761697, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9340558412931668, + "equiv": 4189.212759485188, + "delta_vs_agree2": [ + -0.006520940484937546, + -0.009184423218221896, + -0.004132990448199853 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6168.549855673349, + "delta_vs_agree2": [ + -0.06484202792064658, + -0.07834542615723733, + -0.05198383541513593 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9603232916972814, + "equiv": 5578.010762932879, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9542615723732549, + "equiv": 5002.208335015463, + "delta_vs_agree2": [ + 0.013684790595150624, + 0.010929463629684055, + 0.016623806024981633 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9540778839088905, + "equiv": 5591.969512155735, + "delta_vs_agree2": [ + 0.013501102130786187, + 0.010745775165319618, + 0.0162587252020573 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7115.399391203946, + "delta_vs_agree2": [ + -0.06456649522409992, + -0.07806989346069067, + -0.05152461425422483 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9603232916972814, + "equiv": 5578.010762932879, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9603232916972814, + "equiv": 6728.322036156349, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9572005878030859, + "equiv": 6319.233797062599, + "delta_vs_agree2": [ + 0.016623806024981633, + 0.012950036737692874, + 0.020575404114621592 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8346.294659913927, + "delta_vs_agree2": [ + -0.06447465099191771, + -0.07788850110213079, + -0.05143277002204261 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9603232916972814, + "equiv": 5578.010762932879, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9603232916972814, + "equiv": 6728.322036156349, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9603232916972814, + "equiv": 7878.633309379819, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + }, + "always think": { + "acc": 0.9603232916972814, + "equiv": 3277.388216485939, + "delta_vs_agree2": [ + 0.019746509919177077, + 0.012766348273328435, + 0.026726671565025718 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.8467119764878765, + "equiv": 1677.7417610677344 + }, + { + "R": 8000, + "acc": 0.9099008082292432, + "equiv": 2190.841744717443 + }, + { + "R": 16000, + "acc": 0.9398420279206466, + "equiv": 2525.680790021881 + }, + { + "R": 24000, + "acc": 0.945077149155033, + "equiv": 2701.0134649700044 + }, + { + "R": 32000, + "acc": 0.9544452608376194, + "equiv": 2880.1967099579333 + }, + { + "R": 48000, + "acc": 0.9566495224099927, + "equiv": 3002.5153274902905 + }, + { + "R": 64000, + "acc": 0.9584864070536371, + "equiv": 3126.361602247862 + }, + { + "R": 96000, + "acc": 0.959313005143277, + "equiv": 3268.4759458946637 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9579353416605437, + "equiv": 3528.2999255153372, + "R": [ + 182332.03982533354, + 232322.57610077015 + ], + "delta_vs_agree2": [ + 0.01735855988243938, + 0.012123438648052902, + 0.02305290227773696 + ], + "actions": { + "stop@2": 3251, + "stop@3": 1856, + "stop@5": 282, + "think:q32b_think@3": 121, + "think:q32b_think@0": 3576, + "stop@4": 664, + "stop@6": 62, + "think:q32b_think@8": 31, + "think:q32b_think@7": 45, + "think:q32b_think@6": 38, + "think:q32b_think@5": 116, + "stop@8": 31, + "stop@7": 56, + "think:q32b_think@4": 35, + "think:q32b_think@1": 649, + "think:q32b_think@2": 75 + } + }, + { + "seed": 1, + "acc": 0.9581190301249082, + "equiv": 4443.948748698032, + "R": [ + 377647.0835837676, + 256298.70147382346 + ], + "delta_vs_agree2": [ + 0.01754224834680382, + 0.011204996326230713, + 0.024063188831741366 + ], + "actions": { + "think:q32b_think@0": 3984, + "think:q32b_think@8": 358, + "think:q32b_think@1": 1039, + "stop@2": 3368, + "stop@5": 169, + "stop@3": 426, + "think:q32b_think@3": 141, + "think:q32b_think@2": 230, + "think:q32b_think@4": 161, + "think:q32b_think@5": 119, + "think:q32b_think@6": 59, + "think:q32b_think@7": 46, + "stop@4": 650, + "stop@7": 104, + "stop@8": 10, + "stop@6": 24 + } + }, + { + "seed": 2, + "acc": 0.9585782512858193, + "equiv": 3644.2933288790564, + "R": [ + 195464.0125687913, + 172167.85305362387 + ], + "delta_vs_agree2": [ + 0.018001469507714914, + 0.01230712711241734, + 0.024155033063923585 + ], + "actions": { + "think:q32b_think@2": 268, + "think:q32b_think@3": 254, + "stop@7": 33, + "stop@6": 46, + "stop@5": 237, + "stop@4": 629, + "stop@3": 2999, + "stop@2": 1498, + "think:q32b_think@4": 56, + "think:q32b_think@0": 3600, + "think:q32b_think@7": 11, + "stop@8": 58, + "think:q32b_think@5": 34, + "think:q32b_think@6": 30, + "think:q32b_think@8": 17, + "think:q32b_think@1": 1118 + } + }, + { + "seed": 3, + "acc": 0.9570168993387216, + "equiv": 3346.469277801552, + "R": [ + 176829.00853116572, + 499998.353065514 + ], + "delta_vs_agree2": [ + 0.016440117560617192, + 0.010102865540044085, + 0.02286921381337252 + ], + "actions": { + "stop@2": 2492, + "stop@3": 1572, + "think:q32b_think@1": 870, + "stop@5": 206, + "think:q32b_think@0": 5448, + "think:q32b_think@3": 13, + "stop@6": 39, + "stop@8": 23, + "stop@7": 49, + "think:q32b_think@8": 32, + "think:q32b_think@2": 66, + "think:q32b_think@5": 8, + "think:q32b_think@4": 14, + "think:q32b_think@7": 13, + "think:q32b_think@6": 3, + "stop@4": 40 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9412196914033799, + "equiv": 2580.725200846178, + "R": [ + 21906.703418265966, + 11092.082736867853 + ], + "delta_acc": [ + 0.0006429096252755327, + -0.005696638501102131, + 0.007349834680382059 + ], + "rel_cost": [ + -0.2738442613897872, + -0.3008419126634079, + -0.24720560779686943 + ] + }, + { + "seed": 1, + "acc": 0.9420462894930198, + "equiv": 2564.7022765258253, + "R": [ + 20289.73040181801, + 18849.498095276507 + ], + "delta_acc": [ + 0.0014695077149155032, + -0.004778196179279941, + 0.008268277002204248 + ], + "rel_cost": [ + -0.2783527377051757, + -0.305658830947544, + -0.2520288138861257 + ] + }, + { + "seed": 2, + "acc": 0.9416789125642909, + "equiv": 2563.2277248467917, + "R": [ + 20103.10371473359, + 18384.193553033187 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.005143277002204262, + 0.007533523144746497 + ], + "rel_cost": [ + -0.27876764207517857, + -0.30600929613142014, + -0.25315713056667916 + ] + }, + { + "seed": 3, + "acc": 0.9428728875826599, + "equiv": 2611.7601062970825, + "R": [ + 11205.398916819575, + 19744.7040072149 + ], + "delta_acc": [ + 0.002296105804555474, + -0.0036760653930933136, + 0.008449669360764144 + ], + "rel_cost": [ + -0.2651117645385105, + -0.2921360212478392, + -0.24018601935296416 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.log b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.log new file mode 100644 index 0000000000000000000000000000000000000000..adfa1ae991e7e39537c41dfe90ca6031277fde37 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,532 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> q32b_think): acc 0.941 equiv 3554 +cost model usd: 1 equiv = 4.55e-07 USD; agree2 costs 0.001617 USD per problem + agree2 acc 0.941 equiv 3554 ( +0; 0.001617 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -2404; 0.0005234 USD) | vs agree2 -15.0 [-16.3, -13.7] + SC@2 acc 0.812 equiv 2301 ( -1253; 0.001047 USD) | vs agree2 -12.9 [-14.2, -11.5] + SC@3 acc 0.844 equiv 3451 ( -103; 0.00157 USD) | vs agree2 -9.6 [-10.9, -8.4] + SC@4 acc 0.855 equiv 4601 ( +1047; 0.002094 USD) | vs agree2 -8.5 [-9.8, -7.3] + SC@5 acc 0.863 equiv 5752 ( +2198; 0.002617 USD) | vs agree2 -7.8 [-9.1, -6.5] + SC@6 acc 0.869 equiv 6902 ( +3348; 0.00314 USD) | vs agree2 -7.1 [-8.4, -5.9] + SC@7 acc 0.872 equiv 8052 ( +4498; 0.003664 USD) | vs agree2 -6.9 [-8.2, -5.6] + SC@8 acc 0.876 equiv 9202 ( +5649; 0.004187 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.8) acc 0.871 equiv 4414 ( +860; 0.002008 USD) | vs agree2 -6.9 [-8.2, -5.6] + AC(0.8)+think@2 acc 0.941 equiv 3554 ( +0; 0.001617 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.940 equiv 4038 ( +484; 0.001837 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.934 equiv 4189 ( +635; 0.001906 USD) | vs agree2 -0.7 [-0.9, -0.4] + AC(0.9) acc 0.876 equiv 6169 ( +2615; 0.002807 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.9)+think@2 acc 0.960 equiv 5578 ( +2024; 0.002538 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.9)+think@3 acc 0.954 equiv 5002 ( +1448; 0.002276 USD) | vs agree2 +1.4 [+1.1, +1.7] + AC(0.9)+think@4 acc 0.954 equiv 5592 ( +2038; 0.002544 USD) | vs agree2 +1.4 [+1.1, +1.6] + AC(0.95) acc 0.876 equiv 7115 ( +3561; 0.003238 USD) | vs agree2 -6.5 [-7.8, -5.2] + AC(0.95)+think@2 acc 0.960 equiv 5578 ( +2024; 0.002538 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@3 acc 0.960 equiv 6728 ( +3174; 0.003061 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.95)+think@4 acc 0.957 equiv 6319 ( +2765; 0.002875 USD) | vs agree2 +1.7 [+1.3, +2.1] + AC(0.99) acc 0.876 equiv 8346 ( +4792; 0.003798 USD) | vs agree2 -6.4 [-7.8, -5.1] + AC(0.99)+think@2 acc 0.960 equiv 5578 ( +2024; 0.002538 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@3 acc 0.960 equiv 6728 ( +3174; 0.003061 USD) | vs agree2 +2.0 [+1.3, +2.7] + AC(0.99)+think@4 acc 0.960 equiv 7879 ( +4325; 0.003585 USD) | vs agree2 +2.0 [+1.3, +2.7] + always think acc 0.960 equiv 3277 ( -277; 0.001491 USD) | vs agree2 +2.0 [+1.3, +2.7] +R 4000: acc 0.847 equiv 1678 +R 8000: acc 0.910 equiv 2191 +R 16000: acc 0.940 equiv 2526 +R 24000: acc 0.945 equiv 2701 +R 32000: acc 0.954 equiv 2880 +R 48000: acc 0.957 equiv 3003 +R 64000: acc 0.958 equiv 3126 +R 96000: acc 0.959 equiv 3268 +parity: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 3528 (-26) | vs agree2 +1.7 [+1.2, +2.3] | R [182332, 232323] | {'think:q32b_think@0': 3576, 'stop@2': 3251, 'stop@3': 1856, 'stop@4': 664, 'think:q32b_think@1': 649, 'stop@5': 282, 'think:q32b_think@3': 121, 'think:q32b_think@5': 116} +split 1: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 4444 (+890) | vs agree2 +1.8 [+1.1, +2.4] | R [377647, 256299] | {'think:q32b_think@0': 3984, 'stop@2': 3368, 'think:q32b_think@1': 1039, 'stop@4': 650, 'stop@3': 426, 'think:q32b_think@8': 358, 'think:q32b_think@2': 230, 'stop@5': 169} +split 2: adaptive-SC Bayes at agree2's cost acc 0.959 equiv 3644 (+90) | vs agree2 +1.8 [+1.2, +2.4] | R [195464, 172168] | {'think:q32b_think@0': 3600, 'stop@3': 2999, 'stop@2': 1498, 'think:q32b_think@1': 1118, 'stop@4': 629, 'think:q32b_think@2': 268, 'think:q32b_think@3': 254, 'stop@5': 237} +split 3: adaptive-SC Bayes at agree2's cost acc 0.957 equiv 3346 (-207) | vs agree2 +1.6 [+1.0, +2.3] | R [176829, 499998] | {'think:q32b_think@0': 5448, 'stop@2': 2492, 'stop@3': 1572, 'think:q32b_think@1': 870, 'stop@5': 206, 'think:q32b_think@2': 66, 'stop@7': 49, 'stop@4': 40} +parity at agree2's accuracy: acc 0.941 cost 0.001174 USD vs 0.001617 | cost -27.4% [-30.1, -24.7] | acc +0.1 [-0.6, +0.7] +split 1 at agree2's accuracy: acc 0.942 cost 0.001167 USD vs 0.001617 | cost -27.8% [-30.6, -25.2] | acc +0.1 [-0.5, +0.8] +split 2 at agree2's accuracy: acc 0.942 cost 0.001166 USD vs 0.001617 | cost -27.9% [-30.6, -25.3] | acc +0.1 [-0.5, +0.8] +split 3 at agree2's accuracy: acc 0.943 cost 0.001188 USD vs 0.001617 | cost -26.5% [-29.2, -24.0] | acc +0.2 [-0.4, +0.8] diff --git a/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..cea353c580abf0435da0d44c0819d5607559f40e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc32_usd_openrouter_only.provenance.json @@ -0,0 +1,92 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "openrouter_only", + "--tier", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/sc32_usd_openrouter_only.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 44.9, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..75e02aaa3e70bad826cf8424f36e8454c9893804 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bcf76730b1e7d473f6a6d7ef77ff491829f527c97dc45db6956d7014bb8cc80 +size 327614 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.json b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.json new file mode 100644 index 0000000000000000000000000000000000000000..4461826b54455bf25e7b6caa04c4e1ace34b6809 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.json @@ -0,0 +1,733 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225 + }, + "cost_model": { + "currency": "gpu_s", + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6582624885161, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.3165249770323, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.9747874655477, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.6330499540645, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.29131244258, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.949574931095, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.607837419612, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9197.266099908129, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4412.0116364692885, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5928.330676213297, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5575.824379905499, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.396916034446, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 7354.289774195097, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7941.14717623332, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.669323298678, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 8926.147533635882, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.71692481839, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 12882.837147272214, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 8284.204097318152, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 16000, + "acc": 0.8124540778839089, + "equiv": 1506.2332346595122 + }, + { + "R": 24000, + "acc": 0.9054922850844966, + "equiv": 4730.2022522447505 + }, + { + "R": 32000, + "acc": 0.9287747979426892, + "equiv": 5469.786101770177 + }, + { + "R": 48000, + "acc": 0.9358927259368112, + "equiv": 5813.356175327136 + }, + { + "R": 64000, + "acc": 0.938372520205731, + "equiv": 5983.722530824361 + }, + { + "R": 96000, + "acc": 0.9397961058045555, + "equiv": 6185.602864908272 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9066403379867745, + "equiv": 4780.387792227322, + "R": [ + 31737.957274763416, + 25011.76896809135 + ], + "delta_vs_agree2": [ + -0.023282512858192505, + -0.02902392542248347, + -0.017496326230712712 + ], + "actions": { + "stop@1": 5245, + "stop@2": 2352, + "stop@4": 145, + "think:think16k@0": 1864, + "stop@6": 42, + "stop@5": 179, + "think:think16k@5": 23, + "think:think16k@3": 632, + "think:think16k@6": 10, + "think:think16k@7": 6, + "stop@8": 18, + "stop@7": 34, + "stop@3": 319, + "think:think16k@4": 14, + "think:think16k@8": 5 + } + }, + { + "seed": 1, + "acc": 0.9310249816311535, + "equiv": 5613.162933157068, + "R": [ + 34211.01997690035, + 30133.81637370327 + ], + "delta_vs_agree2": [ + 0.0011021307861866275, + -0.004546289493019839, + 0.006888317413666422 + ], + "actions": { + "stop@2": 2909, + "stop@6": 34, + "stop@5": 118, + "stop@4": 289, + "think:think16k@0": 3688, + "stop@1": 3488, + "think:think16k@5": 72, + "stop@7": 66, + "think:think16k@2": 10, + "stop@3": 62, + "think:think16k@6": 48, + "think:think16k@8": 10, + "think:think16k@3": 70, + "stop@8": 20, + "think:think16k@4": 4 + } + }, + { + "seed": 2, + "acc": 0.9270756796473182, + "equiv": 5450.815983723787, + "R": [ + 34322.76200442399, + 24175.51909619871 + ], + "delta_vs_agree2": [ + -0.002847171197648788, + -0.008496739529757531, + 0.0030779298310066065 + ], + "actions": { + "stop@5": 158, + "stop@4": 134, + "stop@2": 2611, + "stop@1": 3926, + "think:think16k@3": 117, + "think:think16k@2": 42, + "stop@7": 47, + "think:think16k@0": 3600, + "stop@3": 141, + "stop@6": 37, + "think:think16k@8": 6, + "think:think16k@7": 4, + "stop@8": 34, + "think:think16k@4": 13, + "think:think16k@6": 11, + "think:think16k@5": 7 + } + }, + { + "seed": 3, + "acc": 0.9276726671565025, + "equiv": 5433.844433047453, + "R": [ + 29929.841905114903, + 24900.139139552528 + ], + "delta_vs_agree2": [ + -0.0022501836884643643, + -0.008175284717119764, + 0.003950450036737686 + ], + "actions": { + "stop@1": 3963, + "stop@2": 2547, + "think:think16k@3": 94, + "stop@4": 174, + "stop@5": 108, + "stop@6": 55, + "think:think16k@0": 3648, + "stop@3": 181, + "stop@7": 33, + "think:think16k@5": 37, + "stop@8": 18, + "think:think16k@6": 3, + "think:think16k@4": 18, + "think:think16k@8": 4, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9296932402645114, + "equiv": 5526.845547149492, + "R": [ + 31738.04622725488, + 33070.29237209457 + ], + "delta_acc": [ + -0.0002296105804555474, + -0.005923952975753123, + 0.006016945260837613 + ], + "rel_cost": [ + 0.014574388980922596, + -0.016265764902997956, + 0.04722553277173239 + ] + }, + { + "seed": 1, + "acc": 0.9279481998530492, + "equiv": 5501.270490796819, + "R": [ + 24680.402832987907, + 30133.801017737303 + ], + "delta_acc": [ + -0.0019746509919177074, + -0.007763133725202057, + 0.004041146216017634 + ], + "rel_cost": [ + 0.009879523356253932, + -0.022667434990197216, + 0.04355080426891005 + ] + }, + { + "seed": 2, + "acc": 0.9262490815576782, + "equiv": 5406.323231147272, + "R": [ + 31828.864950143754, + 24262.21860421506 + ], + "delta_acc": [ + -0.0036737692872887582, + -0.00936925973548861, + 0.002344324026451126 + ], + "rel_cost": [ + -0.007550140114988803, + -0.039442974188090375, + 0.025170127972970798 + ] + }, + { + "seed": 3, + "acc": 0.9276726671565025, + "equiv": 5433.844433047453, + "R": [ + 29929.86815727641, + 24900.116367032453 + ], + "delta_acc": [ + -0.0022501836884643643, + -0.008175284717119764, + 0.003950450036737686 + ], + "rel_cost": [ + -0.002498016554869073, + -0.03314613177154997, + 0.029405375624232555 + ] + } + ], + "transfer": { + "R": 30789.999548031326, + "m500": { + "acc": 0.932421875, + "equiv": 5548.312209962331, + "delta_vs_agree2": [ + 0.008203125, + -0.0109375, + 0.0296875 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.92421875, + 5429.6649222993865 + ], + "SC@1": [ + 0.78828125, + 1129.850683748888 + ], + "SC@2": [ + 0.81484375, + 2259.701367497776 + ], + "SC@3": [ + 0.84609375, + 3389.5520512466637 + ], + "SC@4": [ + 0.86171875, + 4519.402734995552 + ], + "SC@5": [ + 0.871875, + 5649.25341874444 + ], + "SC@6": [ + 0.8765625, + 6779.104102493327 + ], + "SC@7": [ + 0.88984375, + 7908.954786242216 + ], + "SC@8": [ + 0.89375, + 9038.805469991104 + ], + "AC(0.8)": [ + 0.88828125, + 4191.5573709401015 + ], + "AC(0.8)+think@2": [ + 0.92421875, + 5429.6649222993865 + ], + "AC(0.8)+think@3": [ + 0.92421875, + 5913.907053378809 + ], + "AC(0.8)+think@4": [ + 0.917578125, + 5621.778754294677 + ], + "AC(0.9)": [ + 0.89375, + 6141.172027978451 + ], + "AC(0.9)+think@2": [ + 0.9375, + 10401.37537745888 + ], + "AC(0.9)+think@3": [ + 0.9375, + 7296.6699141805675 + ], + "AC(0.9)+think@4": [ + 0.9375, + 7894.798867542864 + ], + "AC(0.95)": [ + 0.89375, + 7059.039662375095 + ], + "AC(0.95)+think@2": [ + 0.9375, + 10401.37537745888 + ], + "AC(0.95)+think@3": [ + 0.9375, + 11531.226061207766 + ], + "AC(0.95)+think@4": [ + 0.941796875, + 8808.1838031147 + ], + "AC(0.99)": [ + 0.89375, + 8212.999457538364 + ], + "AC(0.99)+think@2": [ + 0.9375, + 10401.37537745888 + ], + "AC(0.99)+think@3": [ + 0.9375, + 11531.226061207766 + ], + "AC(0.99)+think@4": [ + 0.9375, + 12661.076744956656 + ], + "always think": [ + 0.9375, + 8141.6740099611025 + ] + } + }, + "aime": { + "acc": 0.24166666666666667, + "equiv": 3002.457782733884, + "delta_vs_agree2": [ + -0.3125, + -0.40729166666666666, + -0.22083333333333333 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.5541666666666667, + 18915.787878948202 + ], + "SC@1": [ + 0.22708333333333333, + 2576.881539259918 + ], + "SC@2": [ + 0.2625, + 5153.763078519836 + ], + "SC@3": [ + 0.28541666666666665, + 7730.644617779754 + ], + "SC@4": [ + 0.30625, + 10307.526157039672 + ], + "SC@5": [ + 0.3125, + 12884.407696299588 + ], + "SC@6": [ + 0.325, + 15461.289235559509 + ], + "SC@7": [ + 0.33958333333333335, + 18038.17077481943 + ], + "SC@8": [ + 0.35, + 20615.052314079345 + ], + "AC(0.8)": [ + 0.3458333333333333, + 16312.521324942812 + ], + "AC(0.8)+think@2": [ + 0.5541666666666667, + 18915.787878948202 + ], + "AC(0.8)+think@3": [ + 0.5541666666666667, + 21018.56375089806 + ], + "AC(0.8)+think@4": [ + 0.5354166666666667, + 21935.201923610588 + ], + "AC(0.9)": [ + 0.35, + 19032.08558572155 + ], + "AC(0.9)+think@2": [ + 0.5916666666666667, + 22006.16625412832 + ], + "AC(0.9)+think@3": [ + 0.5729166666666666, + 22854.051555274233 + ], + "AC(0.9)+think@4": [ + 0.5729166666666666, + 25169.307578433563 + ], + "AC(0.95)": [ + 0.35, + 19899.130089981674 + ], + "AC(0.95)+think@2": [ + 0.5916666666666667, + 22006.16625412832 + ], + "AC(0.95)+think@3": [ + 0.5916666666666667, + 24583.047793388243 + ], + "AC(0.95)+think@4": [ + 0.5822916666666667, + 26054.21813973169 + ], + "AC(0.99)": [ + 0.35, + 20424.126233160696 + ], + "AC(0.99)+think@2": [ + 0.5916666666666667, + 22006.16625412832 + ], + "AC(0.99)+think@3": [ + 0.5916666666666667, + 24583.047793388243 + ], + "AC(0.99)+think@4": [ + 0.5916666666666667, + 27159.929332648164 + ], + "always think": [ + 0.5916666666666667, + 16852.403175608488 + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.log b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.log new file mode 100644 index 0000000000000000000000000000000000000000..e810207a3d71806747877435a97e978b698b7827 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.log @@ -0,0 +1,54 @@ +2026-10-25 14:12:15,520 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 5447 +cost model gpu_s: 1 equiv = 0.0006341 GPU-seconds; agree2 costs 3.454 GPU-seconds per problem + agree2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -4298; 0.729 GPU-seconds) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -3148; 1.458 GPU-seconds) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1998; 2.187 GPU-seconds) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4599 ( -849; 2.916 GPU-seconds) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +301; 3.645 GPU-seconds) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +1450; 4.374 GPU-seconds) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8048 ( +2600; 5.103 GPU-seconds) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +3750; 5.832 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -1035; 2.798 GPU-seconds) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5928 ( +481; 3.759 GPU-seconds) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5576 ( +128; 3.536 GPU-seconds) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +718; 3.91 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 7354 ( +1907; 4.663 GPU-seconds) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7941 ( +2494; 5.036 GPU-seconds) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7112 ( +1664; 4.51 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 8926 ( +3479; 5.66 GPU-seconds) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8342 ( +2894; 5.29 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 12883 ( +7435; 8.169 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 8284 ( +2837; 5.253 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.812 equiv 1506 +R 24000: acc 0.905 equiv 4730 +R 32000: acc 0.929 equiv 5470 +R 48000: acc 0.936 equiv 5813 +R 64000: acc 0.938 equiv 5984 +R 96000: acc 0.940 equiv 6186 +parity: adaptive-SC Bayes at agree2's cost acc 0.907 equiv 4780 (-667) | vs agree2 -2.3 [-2.9, -1.7] | R [31738, 25012] | {'stop@1': 5245, 'stop@2': 2352, 'think:think16k@0': 1864, 'think:think16k@3': 632, 'stop@3': 319, 'stop@5': 179, 'stop@4': 145, 'stop@6': 42} +split 1: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 5613 (+166) | vs agree2 +0.1 [-0.5, +0.7] | R [34211, 30134] | {'think:think16k@0': 3688, 'stop@1': 3488, 'stop@2': 2909, 'stop@4': 289, 'stop@5': 118, 'think:think16k@5': 72, 'think:think16k@3': 70, 'stop@7': 66} +split 2: adaptive-SC Bayes at agree2's cost acc 0.927 equiv 5451 (+3) | vs agree2 -0.3 [-0.8, +0.3] | R [34323, 24176] | {'stop@1': 3926, 'think:think16k@0': 3600, 'stop@2': 2611, 'stop@5': 158, 'stop@3': 141, 'stop@4': 134, 'think:think16k@3': 117, 'stop@7': 47} +split 3: adaptive-SC Bayes at agree2's cost acc 0.928 equiv 5434 (-14) | vs agree2 -0.2 [-0.8, +0.4] | R [29930, 24900] | {'stop@1': 3963, 'think:think16k@0': 3648, 'stop@2': 2547, 'stop@3': 181, 'stop@4': 174, 'stop@5': 108, 'think:think16k@3': 94, 'stop@6': 55} +parity at agree2's accuracy: acc 0.930 cost 3.505 GPU-seconds vs 3.454 | cost +1.5% [-1.6, +4.7] | acc -0.0 [-0.6, +0.6] +split 1 at agree2's accuracy: acc 0.928 cost 3.488 GPU-seconds vs 3.454 | cost +1.0% [-2.3, +4.4] | acc -0.2 [-0.8, +0.4] +split 2 at agree2's accuracy: acc 0.926 cost 3.428 GPU-seconds vs 3.454 | cost -0.8% [-3.9, +2.5] | acc -0.4 [-0.9, +0.2] +split 3 at agree2's accuracy: acc 0.928 cost 3.446 GPU-seconds vs 3.454 | cost -0.2% [-3.3, +2.9] | acc -0.2 [-0.8, +0.4] +2026-10-25 14:13:10,256 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '\\alpha^{85}-\\alpha^{-85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:20,523 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '-2' +2026-10-25 14:13:30,558 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:40,591 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '123' +2026-10-25 14:13:50,625 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '\\alpha^{85}-\\alpha^{-85}' +2026-10-25 14:14:00,960 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '2' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +2026-10-25 14:14:11,369 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'L_{85}' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +transfer m500: adaptive-SC Bayes acc 0.932 equiv 5548 | agree2 0.924 at 5430 | vs agree2 +0.8 [-1.1, +3.0] | always think 0.938 at 8142 +transfer aime: adaptive-SC Bayes acc 0.242 equiv 3002 | agree2 0.554 at 18916 | vs agree2 -31.2 [-40.7, -22.1] | always think 0.592 at 16852 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..05b4b4747ef0ee108e989d08a6a1a41d3e82a95e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_gpu_s.provenance.json @@ -0,0 +1,93 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "gpu_s", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--match-acc", + "--transfer", + "m500=workdir/selector/pool_m500_k8_chat,workdir/selector/pool_m500_k8_chat/uhead_selector.json,workdir/bayes/m500_tier.json,workdir/bayes/qprior/m500_qprior.json", + "aime=workdir/selector/pool_aime_k8_chat,workdir/selector/pool_aime_k8_chat/uhead_selector.json,workdir/bayes/aime_tier.json,workdir/bayes/qprior/aime_qprior.json", + "--out", + "workdir/bayes/v8/sc8_gpu_s.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:14:16+04:00", + "wall_secs": 123.4, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..085be5e767e3dd9fd5d6ae4cae4b651a25e615ee --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eba426219c106e6a68272c8201ef8b1fd38cbdcf101502d40f23dc61f030461 +size 187669 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.json b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b3826471b3a16490a55d395a851faed09c9df113 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.json @@ -0,0 +1,736 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734 + }, + "cost_model": { + "currency": "tokens", + "unit": "base tokens", + "per_equiv": 1.0, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1124.9408523144746, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2249.881704628949, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3374.822556943424, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4499.763409257898, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5624.704261572373, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6749.645113886848, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 7874.585966201323, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 8999.526818515797, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4328.9513225569435, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5133.4060525349005, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 4968.986232549595, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6046.053545187362, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6364.151193975019, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 6941.938739897135, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 6970.480804555474, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7812.266908523145, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8168.442964731815, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10841.023512123438, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6341.26010286554, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1125.9210139603233 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1126.979803453343 + }, + { + "R": 16000, + "acc": 0.9173861131520941, + "equiv": 4030.2277369581184 + }, + { + "R": 24000, + "acc": 0.9252387950036738, + "equiv": 4310.205464731815 + }, + { + "R": 32000, + "acc": 0.9332292432035268, + "equiv": 4503.716642174872 + }, + { + "R": 48000, + "acc": 0.936627479794269, + "equiv": 4716.755602498163 + }, + { + "R": 64000, + "acc": 0.9396583394562822, + "equiv": 4888.65907421014 + }, + { + "R": 96000, + "acc": 0.9403012490815577, + "equiv": 5089.314639970609 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9354794268919912, + "equiv": 4636.240145113887, + "R": [ + 52046.352193855826, + 35185.86932298981 + ], + "delta_vs_agree2": [ + 0.005556576047024247, + 0.0007347538574577516, + 0.010976533798677437 + ], + "actions": { + "stop@2": 4094, + "stop@4": 122, + "stop@5": 219, + "think:think16k@0": 3576, + "stop@6": 37, + "think:think16k@3": 206, + "think:think16k@8": 6, + "think:think16k@6": 12, + "think:think16k@4": 60, + "stop@8": 19, + "think:think16k@7": 7, + "think:think16k@2": 49, + "stop@7": 61, + "think:think16k@5": 41, + "stop@3": 145, + "stop@1": 2234 + } + }, + { + "seed": 1, + "acc": 0.9348824393828068, + "equiv": 4736.318405584129, + "R": [ + 50402.42040091058, + 43849.97673482865 + ], + "delta_vs_agree2": [ + 0.004959588537839824, + -4.707016899338715e-05, + 0.01047024246877296 + ], + "actions": { + "stop@8": 46, + "stop@7": 46, + "stop@6": 41, + "stop@4": 191, + "stop@3": 93, + "stop@2": 4097, + "think:think16k@0": 3688, + "stop@1": 2121, + "stop@5": 224, + "think:think16k@3": 131, + "think:think16k@4": 62, + "think:think16k@2": 10, + "think:think16k@5": 55, + "think:think16k@7": 23, + "think:think16k@6": 49, + "think:think16k@8": 11 + } + }, + { + "seed": 2, + "acc": 0.9344232182218957, + "equiv": 4668.602066495224, + "R": [ + 46243.34481780861, + 41763.58748877824 + ], + "delta_vs_agree2": [ + 0.004500367376928729, + -0.0005510653930933138, + 0.009690714548126372 + ], + "actions": { + "stop@5": 285, + "stop@4": 74, + "stop@7": 30, + "stop@6": 33, + "stop@3": 78, + "stop@2": 4047, + "stop@1": 2263, + "think:think16k@3": 154, + "think:think16k@2": 152, + "think:think16k@4": 49, + "think:think16k@0": 3600, + "think:think16k@8": 11, + "think:think16k@7": 9, + "stop@8": 62, + "think:think16k@5": 21, + "think:think16k@6": 20 + } + }, + { + "seed": 3, + "acc": 0.936214180749449, + "equiv": 4662.522951873622, + "R": [ + 43408.50571447576, + 42267.05589810954 + ], + "delta_vs_agree2": [ + 0.006291329904481999, + 0.001331741366642175, + 0.011710139603232916 + ], + "actions": { + "stop@1": 2454, + "stop@2": 3669, + "stop@3": 279, + "stop@6": 47, + "think:think16k@3": 137, + "stop@5": 299, + "think:think16k@0": 3648, + "think:think16k@4": 59, + "stop@7": 29, + "think:think16k@5": 33, + "think:think16k@8": 4, + "think:think16k@2": 166, + "think:think16k@6": 20, + "stop@8": 10, + "stop@4": 28, + "think:think16k@7": 6 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9318975018368847, + "equiv": 4448.155271858926, + "R": [ + 29578.903705614226, + 27504.41122046006 + ], + "delta_acc": [ + 0.0019746509919177074, + -0.003123851947097722, + 0.007668993387215283 + ], + "rel_cost": [ + -0.04538346536382752, + -0.07336249426131876, + -0.016401739346024714 + ] + }, + { + "seed": 1, + "acc": 0.9291880969875091, + "equiv": 4460.377470609847, + "R": [ + 23569.059348243198, + 27805.181973248178 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.0061994856722997795, + 0.004913666421748714 + ], + "rel_cost": [ + -0.042760464972845136, + -0.07338813066920981, + -0.0125749532621609 + ] + }, + { + "seed": 2, + "acc": 0.9293717854518736, + "equiv": 4385.907714915504, + "R": [ + 30793.590557203992, + 21673.722457353742 + ], + "delta_acc": [ + -0.0005510653930933138, + -0.0057414125642909625, + 0.005052580822924314 + ], + "rel_cost": [ + -0.05874238461622716, + -0.08726100304692518, + -0.02974360534001899 + ] + }, + { + "seed": 3, + "acc": 0.9310249816311535, + "equiv": 4410.375477590008, + "R": [ + 28154.412857283965, + 23611.696452476262 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.004178912564290963, + 0.006797621234386474 + ], + "rel_cost": [ + -0.05349136944542039, + -0.08083547747240183, + -0.024509551231061723 + ] + } + ], + "transfer": { + "R": 45414.32571178388, + "m500": { + "acc": 0.938671875, + "equiv": 4885.79328125, + "delta_vs_agree2": [ + 0.014453125, + -0.003515625, + 0.03438476562499995 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.92421875, + 4634.017890625 + ], + "SC@1": [ + 0.78828125, + 1103.76328125 + ], + "SC@2": [ + 0.81484375, + 2207.5265625 + ], + "SC@3": [ + 0.84609375, + 3311.28984375 + ], + "SC@4": [ + 0.86171875, + 4415.053125 + ], + "SC@5": [ + 0.871875, + 5518.81640625 + ], + "SC@6": [ + 0.8765625, + 6622.5796875 + ], + "SC@7": [ + 0.88984375, + 7726.34296875 + ], + "SC@8": [ + 0.89375, + 8830.10625 + ], + "AC(0.8)": [ + 0.88828125, + 4099.459375 + ], + "AC(0.8)+think@2": [ + 0.92421875, + 4634.017890625 + ], + "AC(0.8)+think@3": [ + 0.92421875, + 5108.860078125001 + ], + "AC(0.8)+think@4": [ + 0.917578125, + 4986.9370312500005 + ], + "AC(0.9)": [ + 0.89375, + 6009.02890625 + ], + "AC(0.9)+think@2": [ + 0.9375, + 8439.737187499999 + ], + "AC(0.9)+think@3": [ + 0.9375, + 6302.046171874999 + ], + "AC(0.9)+think@4": [ + 0.9375, + 6888.578984375001 + ], + "AC(0.95)": [ + 0.89375, + 6905.77578125 + ], + "AC(0.95)+think@2": [ + 0.9375, + 8439.737187499999 + ], + "AC(0.95)+think@3": [ + 0.9375, + 9543.50046875 + ], + "AC(0.95)+think@4": [ + 0.941796875, + 7697.955703125001 + ], + "AC(0.99)": [ + 0.89375, + 8028.82734375 + ], + "AC(0.99)+think@2": [ + 0.9375, + 8439.737187499999 + ], + "AC(0.99)+think@3": [ + 0.9375, + 9543.50046875 + ], + "AC(0.99)+think@4": [ + 0.9375, + 10647.26375 + ], + "always think": [ + 0.9375, + 6232.210625000001 + ] + } + }, + "aime": { + "acc": 0.5333333333333333, + "equiv": 17110.724166666663, + "delta_vs_agree2": [ + -0.020833333333333332, + -0.04169270833333333, + -0.004166666666666667 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.5541666666666667, + 15601.996666666668 + ], + "SC@1": [ + 0.22708333333333333, + 2533.9333333333334 + ], + "SC@2": [ + 0.2625, + 5067.866666666667 + ], + "SC@3": [ + 0.28541666666666665, + 7601.8 + ], + "SC@4": [ + 0.30625, + 10135.733333333334 + ], + "SC@5": [ + 0.3125, + 12669.666666666666 + ], + "SC@6": [ + 0.325, + 15203.6 + ], + "SC@7": [ + 0.33958333333333335, + 17737.533333333333 + ], + "SC@8": [ + 0.35, + 20271.466666666667 + ], + "AC(0.8)": [ + 0.3458333333333333, + 16049.204166666666 + ], + "AC(0.8)+think@2": [ + 0.5541666666666667, + 15601.996666666668 + ], + "AC(0.8)+think@3": [ + 0.5541666666666667, + 17670.754999999997 + ], + "AC(0.8)+think@4": [ + 0.5354166666666667, + 18834.57125 + ], + "AC(0.9)": [ + 0.35, + 18722.379166666666 + ], + "AC(0.9)+think@2": [ + 0.5916666666666667, + 17967.561666666665 + ], + "AC(0.9)+think@3": [ + 0.5729166666666666, + 19178.001458333332 + ], + "AC(0.9)+think@4": [ + 0.5729166666666666, + 21455.782708333332 + ], + "AC(0.95)": [ + 0.35, + 19571.391666666666 + ], + "AC(0.95)+think@2": [ + 0.5916666666666667, + 17967.561666666665 + ], + "AC(0.95)+think@3": [ + 0.5916666666666667, + 20501.495 + ], + "AC(0.95)+think@4": [ + 0.5822916666666667, + 22189.036666666667 + ], + "AC(0.99)": [ + 0.35, + 20084.541666666668 + ], + "AC(0.99)+think@2": [ + 0.5916666666666667, + 17967.561666666665 + ], + "AC(0.99)+think@3": [ + 0.5916666666666667, + 20501.495 + ], + "AC(0.99)+think@4": [ + 0.5916666666666667, + 23035.428333333333 + ], + "always think": [ + 0.5916666666666667, + 12899.695 + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.log b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.log new file mode 100644 index 0000000000000000000000000000000000000000..9bbecd7b2b76e8e653196eff7b43bd5e4f2687f7 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.log @@ -0,0 +1,54 @@ +2026-10-25 14:12:15,478 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4660 +cost model tokens: 1 equiv = 1 base tokens; agree2 costs 4660 base tokens per problem + agree2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1125 ( -3535; 1125 base tokens) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2250 ( -2410; 2250 base tokens) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3375 ( -1285; 3375 base tokens) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4500 ( -160; 4500 base tokens) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5625 ( +965; 5625 base tokens) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6750 ( +2090; 6750 base tokens) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 7875 ( +3215; 7875 base tokens) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9000 ( +4340; 9000 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4329 ( -331; 4329 base tokens) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5133 ( +474; 5133 base tokens) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 4969 ( +309; 4969 base tokens) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6046 ( +1386; 6046 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6364 ( +1705; 6364 base tokens) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 6942 ( +2282; 6942 base tokens) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 6970 ( +2311; 6970 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7812 ( +3153; 7812 base tokens) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8168 ( +3509; 8168 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10841 ( +6181; 1.084e+04 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6341 ( +1682; 6341 base tokens) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1126 +R 8000: acc 0.791 equiv 1127 +R 16000: acc 0.917 equiv 4030 +R 24000: acc 0.925 equiv 4310 +R 32000: acc 0.933 equiv 4504 +R 48000: acc 0.937 equiv 4717 +R 64000: acc 0.940 equiv 4889 +R 96000: acc 0.940 equiv 5089 +parity: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4636 (-23) | vs agree2 +0.6 [+0.1, +1.1] | R [52046, 35186] | {'stop@2': 4094, 'think:think16k@0': 3576, 'stop@1': 2234, 'stop@5': 219, 'think:think16k@3': 206, 'stop@3': 145, 'stop@4': 122, 'stop@7': 61} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4736 (+77) | vs agree2 +0.5 [-0.0, +1.0] | R [50402, 43850] | {'stop@2': 4097, 'think:think16k@0': 3688, 'stop@1': 2121, 'stop@5': 224, 'stop@4': 191, 'think:think16k@3': 131, 'stop@3': 93, 'think:think16k@4': 62} +split 2: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4669 (+9) | vs agree2 +0.5 [-0.1, +1.0] | R [46243, 41764] | {'stop@2': 4047, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 285, 'think:think16k@3': 154, 'think:think16k@2': 152, 'stop@3': 78, 'stop@4': 74} +split 3: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4663 (+3) | vs agree2 +0.6 [+0.1, +1.2] | R [43409, 42267] | {'stop@2': 3669, 'think:think16k@0': 3648, 'stop@1': 2454, 'stop@5': 299, 'stop@3': 279, 'think:think16k@2': 166, 'think:think16k@3': 137, 'think:think16k@4': 59} +parity at agree2's accuracy: acc 0.932 cost 4448 base tokens vs 4660 | cost -4.5% [-7.3, -1.6] | acc +0.2 [-0.3, +0.8] +split 1 at agree2's accuracy: acc 0.929 cost 4460 base tokens vs 4660 | cost -4.3% [-7.3, -1.3] | acc -0.1 [-0.6, +0.5] +split 2 at agree2's accuracy: acc 0.929 cost 4386 base tokens vs 4660 | cost -5.9% [-8.7, -3.0] | acc -0.1 [-0.6, +0.5] +split 3 at agree2's accuracy: acc 0.931 cost 4410 base tokens vs 4660 | cost -5.3% [-8.1, -2.5] | acc +0.1 [-0.4, +0.7] +2026-10-25 14:13:10,584 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '\\alpha^{85}-\\alpha^{-85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:20,848 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '-2' +2026-10-25 14:13:30,886 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:40,922 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '123' +2026-10-25 14:13:50,957 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '\\alpha^{85}-\\alpha^{-85}' +2026-10-25 14:14:01,298 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '2' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +2026-10-25 14:14:11,714 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'L_{85}' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +transfer m500: adaptive-SC Bayes acc 0.939 equiv 4886 | agree2 0.924 at 4634 | vs agree2 +1.4 [-0.4, +3.4] | always think 0.938 at 6232 +transfer aime: adaptive-SC Bayes acc 0.533 equiv 17111 | agree2 0.554 at 15602 | vs agree2 -2.1 [-4.2, -0.4] | always think 0.592 at 12900 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0322ff990e4ee4bb7d514ccafb7bc383d8cb695d --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_tokens.provenance.json @@ -0,0 +1,93 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "tokens", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json", + "--match-acc", + "--transfer", + "m500=workdir/selector/pool_m500_k8_chat,workdir/selector/pool_m500_k8_chat/uhead_selector.json,workdir/bayes/m500_tier.json,workdir/bayes/qprior/m500_qprior.json", + "aime=workdir/selector/pool_aime_k8_chat,workdir/selector/pool_aime_k8_chat/uhead_selector.json,workdir/bayes/aime_tier.json,workdir/bayes/qprior/aime_qprior.json", + "--out", + "workdir/bayes/v8/sc8_tokens.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:14:17+04:00", + "wall_secs": 123.8, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..c573cde6bea61339f91792fd3d3feff13643cff6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40516fcb4e96d30a73ecaacc02899e75e1af6618ed5d33aecc368295c44f8b43 +size 181636 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.json new file mode 100644 index 0000000000000000000000000000000000000000..72385f3ff771f2b068808f8d6100740c63a5af9c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.json @@ -0,0 +1,736 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "cheapest", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1151.652736958119 + }, + { + "R": 16000, + "acc": 0.9168350477590007, + "equiv": 4046.822556943424 + }, + { + "R": 24000, + "acc": 0.926203159441587, + "equiv": 4346.849260653931 + }, + { + "R": 32000, + "acc": 0.9332292432035268, + "equiv": 4530.6730574944895 + }, + { + "R": 48000, + "acc": 0.936627479794269, + "equiv": 4750.71252755327 + }, + { + "R": 64000, + "acc": 0.9396583394562822, + "equiv": 4924.843451506245 + }, + { + "R": 96000, + "acc": 0.9405767817781043, + "equiv": 5133.706925055107 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9357549595885378, + "equiv": 4665.914447097723, + "R": [ + 53761.60815882869, + 35840.683944266966 + ], + "delta_vs_agree2": [ + 0.0058321087435709035, + 0.0008265980896399706, + 0.011296840558412931 + ], + "actions": { + "stop@1": 2914, + "stop@2": 3300, + "stop@4": 128, + "stop@5": 229, + "stop@3": 232, + "think:think16k@0": 3576, + "stop@6": 35, + "think:think16k@3": 204, + "think:think16k@8": 6, + "think:think16k@6": 10, + "think:think16k@4": 72, + "stop@8": 20, + "think:think16k@7": 8, + "think:think16k@2": 49, + "stop@7": 61, + "think:think16k@5": 44 + } + }, + { + "seed": 1, + "acc": 0.9350661278471712, + "equiv": 4767.403655400441, + "R": [ + 51814.74721102044, + 44596.64305701744 + ], + "delta_vs_agree2": [ + 0.005143277002204262, + 0.00013776634827332844, + 0.010518460690668613 + ], + "actions": { + "stop@8": 43, + "stop@7": 44, + "stop@6": 40, + "stop@4": 191, + "stop@3": 93, + "stop@2": 4097, + "think:think16k@0": 3688, + "stop@1": 2121, + "stop@5": 224, + "think:think16k@3": 135, + "think:think16k@4": 62, + "think:think16k@2": 10, + "think:think16k@5": 55, + "think:think16k@7": 22, + "think:think16k@6": 53, + "think:think16k@8": 10 + } + }, + { + "seed": 2, + "acc": 0.9344232182218957, + "equiv": 4708.9382117928, + "R": [ + 47298.906892707404, + 42459.620096407016 + ], + "delta_vs_agree2": [ + 0.004500367376928729, + -0.0005051432770022043, + 0.009919177075679648 + ], + "actions": { + "stop@5": 271, + "stop@8": 83, + "stop@7": 30, + "stop@6": 34, + "stop@4": 30, + "stop@3": 155, + "stop@2": 3461, + "stop@1": 2778, + "think:think16k@3": 154, + "think:think16k@2": 169, + "think:think16k@4": 50, + "think:think16k@0": 3600, + "think:think16k@8": 14, + "think:think16k@7": 12, + "think:think16k@5": 22, + "think:think16k@6": 25 + } + }, + { + "seed": 3, + "acc": 0.9362601028655401, + "equiv": 4689.776979243204, + "R": [ + 44122.35950080606, + 43636.44372207239 + ], + "delta_vs_agree2": [ + 0.006337252020573108, + 0.0013765154298310068, + 0.011757209772226298 + ], + "actions": { + "stop@1": 2454, + "stop@2": 3669, + "stop@3": 279, + "stop@6": 47, + "think:think16k@3": 145, + "stop@5": 297, + "think:think16k@0": 3648, + "think:think16k@4": 59, + "stop@7": 28, + "think:think16k@5": 35, + "think:think16k@8": 4, + "think:think16k@2": 166, + "think:think16k@6": 16, + "stop@8": 8, + "stop@4": 28, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9318056576047025, + "equiv": 4474.977612968406, + "R": [ + 30117.061370323576, + 27672.9869610281 + ], + "delta_acc": [ + 0.0018828067597354886, + -0.0032145481263776633, + 0.007578297207935335 + ], + "rel_cost": [ + -0.05000932714734109, + -0.07769058467004979, + -0.021280741992484378 + ] + }, + { + "seed": 1, + "acc": 0.9291880969875091, + "equiv": 4485.228577332843, + "R": [ + 24061.26226762234, + 28313.979910682214 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.0061994856722997795, + 0.004913666421748714 + ], + "rel_cost": [ + -0.04783315524745505, + -0.07812418369181799, + -0.017764597058704746 + ] + }, + { + "seed": 2, + "acc": 0.9310249816311535, + "equiv": 4452.599467303453, + "R": [ + 31420.259330627625, + 22070.104606704397 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.003996372152828802, + 0.006613932770022036 + ], + "rel_cost": [ + -0.054759972065839574, + -0.08278413995488167, + -0.026639383846745635 + ] + }, + { + "seed": 3, + "acc": 0.9310709037472447, + "equiv": 4437.428430382072, + "R": [ + 28652.598441905946, + 24064.307585194245 + ], + "delta_acc": [ + 0.001148052902277737, + -0.004087068332108743, + 0.006842395297575312 + ], + "rel_cost": [ + -0.05798062361302248, + -0.08510425518256229, + -0.02927755089924474 + ] + } + ], + "transfer": { + "R": 46838.755346627, + "m500": { + "acc": 0.938671875, + "equiv": 4945.41875, + "delta_vs_agree2": [ + 0.014453125, + -0.003515625, + 0.03438476562499995 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.92421875, + 4687.9595703125 + ], + "SC@1": [ + 0.78828125, + 1129.79609375 + ], + "SC@2": [ + 0.81484375, + 2259.5921875 + ], + "SC@3": [ + 0.84609375, + 3389.38828125 + ], + "SC@4": [ + 0.86171875, + 4519.184375 + ], + "SC@5": [ + 0.871875, + 5648.98046875 + ], + "SC@6": [ + 0.8765625, + 6778.7765625 + ], + "SC@7": [ + 0.88984375, + 7908.57265625 + ], + "SC@8": [ + 0.89375, + 9038.36875 + ], + "AC(0.8)": [ + 0.88828125, + 4191.3646484375 + ], + "AC(0.8)+think@2": [ + 0.92421875, + 4687.9595703125 + ], + "AC(0.8)+think@3": [ + 0.92421875, + 5172.18203125 + ], + "AC(0.8)+think@4": [ + 0.917578125, + 5059.1787109375 + ], + "AC(0.9)": [ + 0.89375, + 6140.8955078125 + ], + "AC(0.9)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.9)+think@3": [ + 0.9375, + 6382.458984375 + ], + "AC(0.9)+think@4": [ + 0.9375, + 6980.563671875 + ], + "AC(0.95)": [ + 0.89375, + 7058.7189453125 + ], + "AC(0.95)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.95)+think@3": [ + 0.9375, + 9626.80546875 + ], + "AC(0.95)+think@4": [ + 0.941796875, + 7804.606640625 + ], + "AC(0.99)": [ + 0.89375, + 8212.6140625 + ], + "AC(0.99)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.99)+think@3": [ + 0.9375, + 9626.80546875 + ], + "AC(0.99)+think@4": [ + 0.9375, + 10756.6015625 + ], + "always think": [ + 0.9375, + 6237.4171875 + ] + } + }, + "aime": { + "acc": 0.5333333333333333, + "equiv": 17242.7484375, + "delta_vs_agree2": [ + -0.020833333333333332, + -0.04169270833333333, + -0.004166666666666667 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.5541666666666667, + 15694.502604166666 + ], + "SC@1": [ + 0.22708333333333333, + 2576.7916666666665 + ], + "SC@2": [ + 0.2625, + 5153.583333333333 + ], + "SC@3": [ + 0.28541666666666665, + 7730.375 + ], + "SC@4": [ + 0.30625, + 10307.166666666666 + ], + "SC@5": [ + 0.3125, + 12883.958333333334 + ], + "SC@6": [ + 0.325, + 15460.75 + ], + "SC@7": [ + 0.33958333333333335, + 18037.541666666668 + ], + "SC@8": [ + 0.35, + 20614.333333333332 + ], + "AC(0.8)": [ + 0.3458333333333333, + 16311.9703125 + ], + "AC(0.8)+think@2": [ + 0.5541666666666667, + 15694.502604166666 + ], + "AC(0.8)+think@3": [ + 0.5541666666666667, + 17797.207291666666 + ], + "AC(0.8)+think@4": [ + 0.5354166666666667, + 18994.310416666667 + ], + "AC(0.9)": [ + 0.35, + 19031.4375 + ], + "AC(0.9)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.9)+think@3": [ + 0.5729166666666666, + 19314.055729166666 + ], + "AC(0.9)+think@4": [ + 0.5729166666666666, + 21629.233333333334 + ], + "AC(0.95)": [ + 0.35, + 19898.444270833334 + ], + "AC(0.95)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.95)+think@3": [ + 0.5916666666666667, + 20638.641666666666 + ], + "AC(0.95)+think@4": [ + 0.5822916666666667, + 22368.317708333332 + ], + "AC(0.99)": [ + 0.35, + 20423.415625 + ], + "AC(0.99)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.99)+think@3": [ + 0.5916666666666667, + 20638.641666666666 + ], + "AC(0.99)+think@4": [ + 0.5916666666666667, + 23215.433333333334 + ], + "always think": [ + 0.5916666666666667, + 12908.266666666666 + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.log b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.log new file mode 100644 index 0000000000000000000000000000000000000000..84dbce3ec2ff5566a3da8b4207076f8d9748faab --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.log @@ -0,0 +1,54 @@ +2026-10-25 14:12:15,510 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1152 +R 16000: acc 0.917 equiv 4047 +R 24000: acc 0.926 equiv 4347 +R 32000: acc 0.933 equiv 4531 +R 48000: acc 0.937 equiv 4751 +R 64000: acc 0.940 equiv 4925 +R 96000: acc 0.941 equiv 5134 +parity: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4666 (-45) | vs agree2 +0.6 [+0.1, +1.1] | R [53762, 35841] | {'think:think16k@0': 3576, 'stop@2': 3300, 'stop@1': 2914, 'stop@3': 232, 'stop@5': 229, 'think:think16k@3': 204, 'stop@4': 128, 'think:think16k@4': 72} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4767 (+57) | vs agree2 +0.5 [+0.0, +1.1] | R [51815, 44597] | {'stop@2': 4097, 'think:think16k@0': 3688, 'stop@1': 2121, 'stop@5': 224, 'stop@4': 191, 'think:think16k@3': 135, 'stop@3': 93, 'think:think16k@4': 62} +split 2: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4709 (-2) | vs agree2 +0.5 [-0.1, +1.0] | R [47299, 42460] | {'think:think16k@0': 3600, 'stop@2': 3461, 'stop@1': 2778, 'stop@5': 271, 'think:think16k@2': 169, 'stop@3': 155, 'think:think16k@3': 154, 'stop@8': 83} +split 3: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4690 (-21) | vs agree2 +0.6 [+0.1, +1.2] | R [44122, 43636] | {'stop@2': 3669, 'think:think16k@0': 3648, 'stop@1': 2454, 'stop@5': 297, 'stop@3': 279, 'think:think16k@2': 166, 'think:think16k@3': 145, 'think:think16k@4': 59} +parity at agree2's accuracy: acc 0.932 cost 0.000895 USD vs 0.0009421 | cost -5.0% [-7.8, -2.1] | acc +0.2 [-0.3, +0.8] +split 1 at agree2's accuracy: acc 0.929 cost 0.000897 USD vs 0.0009421 | cost -4.8% [-7.8, -1.8] | acc -0.1 [-0.6, +0.5] +split 2 at agree2's accuracy: acc 0.931 cost 0.0008905 USD vs 0.0009421 | cost -5.5% [-8.3, -2.7] | acc +0.1 [-0.4, +0.7] +split 3 at agree2's accuracy: acc 0.931 cost 0.0008875 USD vs 0.0009421 | cost -5.8% [-8.5, -2.9] | acc +0.1 [-0.4, +0.7] +2026-10-25 14:13:11,000 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '\\alpha^{85}-\\alpha^{-85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:21,267 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '-2' +2026-10-25 14:13:31,304 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:41,346 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '123' +2026-10-25 14:13:51,386 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '\\alpha^{85}-\\alpha^{-85}' +2026-10-25 14:14:01,733 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '2' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +2026-10-25 14:14:12,150 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'L_{85}' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +transfer m500: adaptive-SC Bayes acc 0.939 equiv 4945 | agree2 0.924 at 4688 | vs agree2 +1.4 [-0.4, +3.4] | always think 0.938 at 6237 +transfer aime: adaptive-SC Bayes acc 0.533 equiv 17243 | agree2 0.554 at 15695 | vs agree2 -2.1 [-4.2, -0.4] | always think 0.592 at 12908 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0265b188ea221432afca14d0b377f336fd330567 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_cheapest.provenance.json @@ -0,0 +1,95 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "cheapest", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--match-acc", + "--transfer", + "m500=workdir/selector/pool_m500_k8_chat,workdir/selector/pool_m500_k8_chat/uhead_selector.json,workdir/bayes/m500_tier.json,workdir/bayes/qprior/m500_qprior.json", + "aime=workdir/selector/pool_aime_k8_chat,workdir/selector/pool_aime_k8_chat/uhead_selector.json,workdir/bayes/aime_tier.json,workdir/bayes/qprior/aime_qprior.json", + "--out", + "workdir/bayes/v8/sc8_usd_cheapest.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:14:17+04:00", + "wall_secs": 124.2, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..c573cde6bea61339f91792fd3d3feff13643cff6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40516fcb4e96d30a73ecaacc02899e75e1af6618ed5d33aecc368295c44f8b43 +size 181636 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5f22cca418aef5f3378f6710d4ef3bb619d5eb --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.json @@ -0,0 +1,736 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "dear_32b", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1151.652736958119 + }, + { + "R": 16000, + "acc": 0.9168350477590007, + "equiv": 4046.822556943424 + }, + { + "R": 24000, + "acc": 0.926203159441587, + "equiv": 4346.849260653931 + }, + { + "R": 32000, + "acc": 0.9332292432035268, + "equiv": 4530.6730574944895 + }, + { + "R": 48000, + "acc": 0.936627479794269, + "equiv": 4750.71252755327 + }, + { + "R": 64000, + "acc": 0.9396583394562822, + "equiv": 4924.843451506245 + }, + { + "R": 96000, + "acc": 0.9405767817781043, + "equiv": 5133.706925055107 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9357549595885378, + "equiv": 4665.914447097723, + "R": [ + 53761.60815882869, + 35840.683944266966 + ], + "delta_vs_agree2": [ + 0.0058321087435709035, + 0.0008265980896399706, + 0.011296840558412931 + ], + "actions": { + "stop@1": 2914, + "stop@2": 3300, + "stop@4": 128, + "stop@5": 229, + "stop@3": 232, + "think:think16k@0": 3576, + "stop@6": 35, + "think:think16k@3": 204, + "think:think16k@8": 6, + "think:think16k@6": 10, + "think:think16k@4": 72, + "stop@8": 20, + "think:think16k@7": 8, + "think:think16k@2": 49, + "stop@7": 61, + "think:think16k@5": 44 + } + }, + { + "seed": 1, + "acc": 0.9350661278471712, + "equiv": 4767.403655400441, + "R": [ + 51814.74721102044, + 44596.64305701744 + ], + "delta_vs_agree2": [ + 0.005143277002204262, + 0.00013776634827332844, + 0.010518460690668613 + ], + "actions": { + "stop@8": 43, + "stop@7": 44, + "stop@6": 40, + "stop@4": 191, + "stop@3": 93, + "stop@2": 4097, + "think:think16k@0": 3688, + "stop@1": 2121, + "stop@5": 224, + "think:think16k@3": 135, + "think:think16k@4": 62, + "think:think16k@2": 10, + "think:think16k@5": 55, + "think:think16k@7": 22, + "think:think16k@6": 53, + "think:think16k@8": 10 + } + }, + { + "seed": 2, + "acc": 0.9344232182218957, + "equiv": 4708.9382117928, + "R": [ + 47298.906892707404, + 42459.620096407016 + ], + "delta_vs_agree2": [ + 0.004500367376928729, + -0.0005051432770022043, + 0.009919177075679648 + ], + "actions": { + "stop@5": 271, + "stop@8": 83, + "stop@7": 30, + "stop@6": 34, + "stop@4": 30, + "stop@3": 155, + "stop@2": 3461, + "stop@1": 2778, + "think:think16k@3": 154, + "think:think16k@2": 169, + "think:think16k@4": 50, + "think:think16k@0": 3600, + "think:think16k@8": 14, + "think:think16k@7": 12, + "think:think16k@5": 22, + "think:think16k@6": 25 + } + }, + { + "seed": 3, + "acc": 0.9362601028655401, + "equiv": 4689.776979243204, + "R": [ + 44122.35950080606, + 43636.44372207239 + ], + "delta_vs_agree2": [ + 0.006337252020573108, + 0.0013765154298310068, + 0.011757209772226298 + ], + "actions": { + "stop@1": 2454, + "stop@2": 3669, + "stop@3": 279, + "stop@6": 47, + "think:think16k@3": 145, + "stop@5": 297, + "think:think16k@0": 3648, + "think:think16k@4": 59, + "stop@7": 28, + "think:think16k@5": 35, + "think:think16k@8": 4, + "think:think16k@2": 166, + "think:think16k@6": 16, + "stop@8": 8, + "stop@4": 28, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9318056576047025, + "equiv": 4474.977612968406, + "R": [ + 30117.061370323576, + 27672.9869610281 + ], + "delta_acc": [ + 0.0018828067597354886, + -0.0032145481263776633, + 0.007578297207935335 + ], + "rel_cost": [ + -0.05000932714734109, + -0.07769058467004979, + -0.021280741992484378 + ] + }, + { + "seed": 1, + "acc": 0.9291880969875091, + "equiv": 4485.228577332843, + "R": [ + 24061.26226762234, + 28313.979910682214 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.0061994856722997795, + 0.004913666421748714 + ], + "rel_cost": [ + -0.04783315524745505, + -0.07812418369181799, + -0.017764597058704746 + ] + }, + { + "seed": 2, + "acc": 0.9310249816311535, + "equiv": 4452.599467303453, + "R": [ + 31420.259330627625, + 22070.104606704397 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.003996372152828802, + 0.006613932770022036 + ], + "rel_cost": [ + -0.054759972065839574, + -0.08278413995488167, + -0.026639383846745635 + ] + }, + { + "seed": 3, + "acc": 0.9310709037472447, + "equiv": 4437.428430382072, + "R": [ + 28652.598441905946, + 24064.307585194245 + ], + "delta_acc": [ + 0.001148052902277737, + -0.004087068332108743, + 0.006842395297575312 + ], + "rel_cost": [ + -0.05798062361302248, + -0.08510425518256229, + -0.02927755089924474 + ] + } + ], + "transfer": { + "R": 46838.755346627, + "m500": { + "acc": 0.938671875, + "equiv": 4945.41875, + "delta_vs_agree2": [ + 0.014453125, + -0.003515625, + 0.03438476562499995 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.92421875, + 4687.9595703125 + ], + "SC@1": [ + 0.78828125, + 1129.79609375 + ], + "SC@2": [ + 0.81484375, + 2259.5921875 + ], + "SC@3": [ + 0.84609375, + 3389.38828125 + ], + "SC@4": [ + 0.86171875, + 4519.184375 + ], + "SC@5": [ + 0.871875, + 5648.98046875 + ], + "SC@6": [ + 0.8765625, + 6778.7765625 + ], + "SC@7": [ + 0.88984375, + 7908.57265625 + ], + "SC@8": [ + 0.89375, + 9038.36875 + ], + "AC(0.8)": [ + 0.88828125, + 4191.3646484375 + ], + "AC(0.8)+think@2": [ + 0.92421875, + 4687.9595703125 + ], + "AC(0.8)+think@3": [ + 0.92421875, + 5172.18203125 + ], + "AC(0.8)+think@4": [ + 0.917578125, + 5059.1787109375 + ], + "AC(0.9)": [ + 0.89375, + 6140.8955078125 + ], + "AC(0.9)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.9)+think@3": [ + 0.9375, + 6382.458984375 + ], + "AC(0.9)+think@4": [ + 0.9375, + 6980.563671875 + ], + "AC(0.95)": [ + 0.89375, + 7058.7189453125 + ], + "AC(0.95)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.95)+think@3": [ + 0.9375, + 9626.80546875 + ], + "AC(0.95)+think@4": [ + 0.941796875, + 7804.606640625 + ], + "AC(0.99)": [ + 0.89375, + 8212.6140625 + ], + "AC(0.99)+think@2": [ + 0.9375, + 8497.009375 + ], + "AC(0.99)+think@3": [ + 0.9375, + 9626.80546875 + ], + "AC(0.99)+think@4": [ + 0.9375, + 10756.6015625 + ], + "always think": [ + 0.9375, + 6237.4171875 + ] + } + }, + "aime": { + "acc": 0.5333333333333333, + "equiv": 17242.7484375, + "delta_vs_agree2": [ + -0.020833333333333332, + -0.04169270833333333, + -0.004166666666666667 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.5541666666666667, + 15694.502604166666 + ], + "SC@1": [ + 0.22708333333333333, + 2576.7916666666665 + ], + "SC@2": [ + 0.2625, + 5153.583333333333 + ], + "SC@3": [ + 0.28541666666666665, + 7730.375 + ], + "SC@4": [ + 0.30625, + 10307.166666666666 + ], + "SC@5": [ + 0.3125, + 12883.958333333334 + ], + "SC@6": [ + 0.325, + 15460.75 + ], + "SC@7": [ + 0.33958333333333335, + 18037.541666666668 + ], + "SC@8": [ + 0.35, + 20614.333333333332 + ], + "AC(0.8)": [ + 0.3458333333333333, + 16311.9703125 + ], + "AC(0.8)+think@2": [ + 0.5541666666666667, + 15694.502604166666 + ], + "AC(0.8)+think@3": [ + 0.5541666666666667, + 17797.207291666666 + ], + "AC(0.8)+think@4": [ + 0.5354166666666667, + 18994.310416666667 + ], + "AC(0.9)": [ + 0.35, + 19031.4375 + ], + "AC(0.9)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.9)+think@3": [ + 0.5729166666666666, + 19314.055729166666 + ], + "AC(0.9)+think@4": [ + 0.5729166666666666, + 21629.233333333334 + ], + "AC(0.95)": [ + 0.35, + 19898.444270833334 + ], + "AC(0.95)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.95)+think@3": [ + 0.5916666666666667, + 20638.641666666666 + ], + "AC(0.95)+think@4": [ + 0.5822916666666667, + 22368.317708333332 + ], + "AC(0.99)": [ + 0.35, + 20423.415625 + ], + "AC(0.99)+think@2": [ + 0.5916666666666667, + 18061.85 + ], + "AC(0.99)+think@3": [ + 0.5916666666666667, + 20638.641666666666 + ], + "AC(0.99)+think@4": [ + 0.5916666666666667, + 23215.433333333334 + ], + "always think": [ + 0.5916666666666667, + 12908.266666666666 + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.log b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.log new file mode 100644 index 0000000000000000000000000000000000000000..19c6a1a129b2e9072011cc709b9618083c38292d --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.log @@ -0,0 +1,54 @@ +2026-10-25 14:12:15,516 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1152 +R 16000: acc 0.917 equiv 4047 +R 24000: acc 0.926 equiv 4347 +R 32000: acc 0.933 equiv 4531 +R 48000: acc 0.937 equiv 4751 +R 64000: acc 0.940 equiv 4925 +R 96000: acc 0.941 equiv 5134 +parity: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4666 (-45) | vs agree2 +0.6 [+0.1, +1.1] | R [53762, 35841] | {'think:think16k@0': 3576, 'stop@2': 3300, 'stop@1': 2914, 'stop@3': 232, 'stop@5': 229, 'think:think16k@3': 204, 'stop@4': 128, 'think:think16k@4': 72} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4767 (+57) | vs agree2 +0.5 [+0.0, +1.1] | R [51815, 44597] | {'stop@2': 4097, 'think:think16k@0': 3688, 'stop@1': 2121, 'stop@5': 224, 'stop@4': 191, 'think:think16k@3': 135, 'stop@3': 93, 'think:think16k@4': 62} +split 2: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4709 (-2) | vs agree2 +0.5 [-0.1, +1.0] | R [47299, 42460] | {'think:think16k@0': 3600, 'stop@2': 3461, 'stop@1': 2778, 'stop@5': 271, 'think:think16k@2': 169, 'stop@3': 155, 'think:think16k@3': 154, 'stop@8': 83} +split 3: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4690 (-21) | vs agree2 +0.6 [+0.1, +1.2] | R [44122, 43636] | {'stop@2': 3669, 'think:think16k@0': 3648, 'stop@1': 2454, 'stop@5': 297, 'stop@3': 279, 'think:think16k@2': 166, 'think:think16k@3': 145, 'think:think16k@4': 59} +parity at agree2's accuracy: acc 0.932 cost 0.000895 USD vs 0.0009421 | cost -5.0% [-7.8, -2.1] | acc +0.2 [-0.3, +0.8] +split 1 at agree2's accuracy: acc 0.929 cost 0.000897 USD vs 0.0009421 | cost -4.8% [-7.8, -1.8] | acc -0.1 [-0.6, +0.5] +split 2 at agree2's accuracy: acc 0.931 cost 0.0008905 USD vs 0.0009421 | cost -5.5% [-8.3, -2.7] | acc +0.1 [-0.4, +0.7] +split 3 at agree2's accuracy: acc 0.931 cost 0.0008875 USD vs 0.0009421 | cost -5.8% [-8.5, -2.9] | acc +0.1 [-0.4, +0.7] +2026-10-25 14:13:10,652 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '\\alpha^{85}-\\alpha^{-85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:20,921 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '-2' +2026-10-25 14:13:30,954 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:40,988 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '123' +2026-10-25 14:13:51,023 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '\\alpha^{85}-\\alpha^{-85}' +2026-10-25 14:14:01,370 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '2' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +2026-10-25 14:14:11,791 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'L_{85}' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +transfer m500: adaptive-SC Bayes acc 0.939 equiv 4945 | agree2 0.924 at 4688 | vs agree2 +1.4 [-0.4, +3.4] | always think 0.938 at 6237 +transfer aime: adaptive-SC Bayes acc 0.533 equiv 17243 | agree2 0.554 at 15695 | vs agree2 -2.1 [-4.2, -0.4] | always think 0.592 at 12908 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..63d31f160de4402d45f1f02061de3f26d57f6665 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_dear_32b.provenance.json @@ -0,0 +1,95 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "dear_32b", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--match-acc", + "--transfer", + "m500=workdir/selector/pool_m500_k8_chat,workdir/selector/pool_m500_k8_chat/uhead_selector.json,workdir/bayes/m500_tier.json,workdir/bayes/qprior/m500_qprior.json", + "aime=workdir/selector/pool_aime_k8_chat,workdir/selector/pool_aime_k8_chat/uhead_selector.json,workdir/bayes/aime_tier.json,workdir/bayes/qprior/aime_qprior.json", + "--out", + "workdir/bayes/v8/sc8_usd_dear_32b.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:14:17+04:00", + "wall_secs": 123.8, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..417ad92f09ed832429f0cf7788eff813cf852fc6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39cc295232f8bbbdacea55926256031fb2be712aacd12b06b4843aedd428997f +size 227117 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.json new file mode 100644 index 0000000000000000000000000000000000000000..a704792615d2808e0d0460f0f02730dd581c82cb --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.json @@ -0,0 +1,736 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "scenario": "openrouter_only", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1150.3112732234702, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2300.6225464469403, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3450.9338196704107, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4601.245092893881, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5751.556366117351, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6901.867639340821, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8052.178912564291, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9202.490185787761, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4414.206011861026, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5194.155375511704, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5037.402507347539, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6168.549855673349, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6442.532221580771, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7030.53273328435, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7115.399391203946, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7916.276974650992, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8346.294659913927, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10948.143067072531, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.89797417865, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1151.3006455337463 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1152.3577267240476 + }, + { + "R": 16000, + "acc": 0.9168350477590007, + "equiv": 4047.6503345754168 + }, + { + "R": 24000, + "acc": 0.926203159441587, + "equiv": 4347.806805657605 + }, + { + "R": 32000, + "acc": 0.9332292432035268, + "equiv": 4531.720880130157 + }, + { + "R": 48000, + "acc": 0.936627479794269, + "equiv": 4751.935038574577 + }, + { + "R": 64000, + "acc": 0.9396583394562822, + "equiv": 4926.162606276897 + }, + { + "R": 96000, + "acc": 0.9405767817781043, + "equiv": 5135.149158969245 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.93584680382072, + "equiv": 4693.12120289703, + "R": [ + 53810.86005302637, + 35859.57767630928 + ], + "delta_vs_agree2": [ + 0.005923952975753123, + 0.0011021307861866275, + 0.011389832843497424 + ], + "actions": { + "stop@2": 3983, + "stop@4": 126, + "stop@5": 225, + "stop@3": 232, + "think:think16k@0": 3576, + "stop@6": 39, + "think:think16k@3": 206, + "think:think16k@8": 6, + "think:think16k@6": 13, + "think:think16k@4": 66, + "stop@8": 19, + "think:think16k@7": 8, + "think:think16k@2": 49, + "stop@7": 62, + "think:think16k@5": 44, + "stop@1": 2234 + } + }, + { + "seed": 1, + "acc": 0.9351579720793534, + "equiv": 4782.876062768972, + "R": [ + 51856.066123999015, + 44618.97680005004 + ], + "delta_vs_agree2": [ + 0.00523512123438648, + 0.0001836884643644379, + 0.010609156869948561 + ], + "actions": { + "stop@8": 72, + "stop@7": 46, + "stop@6": 45, + "stop@4": 146, + "stop@3": 93, + "stop@2": 4097, + "think:think16k@0": 3688, + "stop@1": 2121, + "stop@5": 232, + "think:think16k@3": 135, + "think:think16k@4": 62, + "think:think16k@2": 10, + "think:think16k@5": 55, + "think:think16k@7": 22, + "think:think16k@6": 53, + "think:think16k@8": 11 + } + }, + { + "seed": 2, + "acc": 0.934974283614989, + "equiv": 4754.988396137294, + "R": [ + 47340.055098821744, + 42480.60377813101 + ], + "delta_vs_agree2": [ + 0.0050514327700220425, + -1.1480529022776717e-06, + 0.010240631888317414 + ], + "actions": { + "stop@5": 288, + "stop@8": 86, + "stop@7": 31, + "stop@6": 38, + "stop@4": 30, + "stop@3": 155, + "stop@2": 3947, + "stop@1": 2263, + "think:think16k@3": 154, + "think:think16k@2": 169, + "think:think16k@4": 50, + "think:think16k@0": 3600, + "think:think16k@8": 17, + "think:think16k@7": 13, + "think:think16k@5": 22, + "think:think16k@6": 25 + } + }, + { + "seed": 3, + "acc": 0.9350202057310801, + "equiv": 4602.485117298205, + "R": [ + 44143.583296220444, + 43675.556607144805 + ], + "delta_vs_agree2": [ + 0.005097354886113152, + 0.0001836884643644379, + 0.01056208670095518 + ], + "actions": { + "stop@1": 3492, + "stop@2": 2759, + "think:think16k@3": 145, + "stop@5": 267, + "stop@6": 40, + "think:think16k@0": 3648, + "think:think16k@4": 55, + "stop@7": 23, + "think:think16k@5": 27, + "think:think16k@8": 4, + "think:think16k@2": 166, + "think:think16k@6": 15, + "stop@8": 7, + "stop@4": 28, + "stop@3": 207, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9318056576047025, + "equiv": 4476.0008410307555, + "R": [ + 30132.96679376429, + 27679.171249374267 + ], + "delta_acc": [ + 0.0018828067597354886, + -0.0032145481263776633, + 0.007578297207935335 + ], + "rel_cost": [ + -0.05012219713906834, + -0.07779169724691741, + -0.021399806701248466 + ] + }, + { + "seed": 1, + "acc": 0.9291880969875091, + "equiv": 4486.223815209405, + "R": [ + 24075.635742046656, + 28329.045129315553 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.0061994856722997795, + 0.004913666421748714 + ], + "rel_cost": [ + -0.04795272117237391, + -0.0782325235396055, + -0.0178891766296749 + ] + }, + { + "seed": 2, + "acc": 0.9310249816311535, + "equiv": 4453.5877414191245, + "R": [ + 31438.158490679172, + 22081.80393681677 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.003996372152828802, + 0.006613932770022036 + ], + "rel_cost": [ + -0.05487860951934299, + -0.0829032578129803, + -0.02675916535456123 + ] + }, + { + "seed": 3, + "acc": 0.9310709037472447, + "equiv": 4438.417143381967, + "R": [ + 28667.390320181832, + 24077.587769637244 + ], + "delta_acc": [ + 0.001148052902277737, + -0.004087068332108743, + 0.006842395297575312 + ], + "rel_cost": [ + -0.05809804911361776, + -0.08520668151297489, + -0.029401541645898095 + ] + } + ], + "transfer": { + "R": 46879.50323882489, + "m500": { + "acc": 0.938671875, + "equiv": 4946.747075892857, + "delta_vs_agree2": [ + 0.014453125, + -0.003515625, + 0.03438476562499995 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.92421875, + 4689.7151674107145 + ], + "SC@1": [ + 0.78828125, + 1130.539888392857 + ], + "SC@2": [ + 0.81484375, + 2261.079776785714 + ], + "SC@3": [ + 0.84609375, + 3391.619665178571 + ], + "SC@4": [ + 0.86171875, + 4522.159553571428 + ], + "SC@5": [ + 0.871875, + 5652.699441964285 + ], + "SC@6": [ + 0.8765625, + 6783.239330357142 + ], + "SC@7": [ + 0.88984375, + 7913.77921875 + ], + "SC@8": [ + 0.89375, + 9044.319107142856 + ], + "AC(0.8)": [ + 0.88828125, + 4193.990513392857 + ], + "AC(0.8)+think@2": [ + 0.92421875, + 4689.7151674107145 + ], + "AC(0.8)+think@3": [ + 0.92421875, + 5174.205636160715 + ], + "AC(0.8)+think@4": [ + 0.917578125, + 5061.40453125 + ], + "AC(0.9)": [ + 0.89375, + 6144.663125 + ], + "AC(0.9)+think@2": [ + 0.9375, + 8499.240758928572 + ], + "AC(0.9)+think@3": [ + 0.9375, + 6385.020993303571 + ], + "AC(0.9)+think@4": [ + 0.9375, + 6983.456305803571 + ], + "AC(0.95)": [ + 0.89375, + 7063.08875 + ], + "AC(0.95)+think@2": [ + 0.9375, + 8499.240758928572 + ], + "AC(0.95)+think@3": [ + 0.9375, + 9629.78064732143 + ], + "AC(0.95)+think@4": [ + 0.941796875, + 7807.94177455357 + ], + "AC(0.99)": [ + 0.89375, + 8217.865111607143 + ], + "AC(0.99)+think@2": [ + 0.9375, + 8499.240758928572 + ], + "AC(0.99)+think@3": [ + 0.9375, + 9629.78064732143 + ], + "AC(0.99)+think@4": [ + 0.9375, + 10760.320535714285 + ], + "always think": [ + 0.9375, + 6238.160982142857 + ] + } + }, + "aime": { + "acc": 0.5333333333333333, + "equiv": 17247.202619047617, + "delta_vs_agree2": [ + -0.020833333333333332, + -0.04169270833333333, + -0.004166666666666667 + ], + "prior_shift": null, + "delta": 0.0, + "baselines": { + "agree2": [ + 0.5541666666666667, + 15697.92154761905 + ], + "SC@1": [ + 0.22708333333333333, + 2578.016190476191 + ], + "SC@2": [ + 0.2625, + 5156.032380952382 + ], + "SC@3": [ + 0.28541666666666665, + 7734.048571428571 + ], + "SC@4": [ + 0.30625, + 10312.064761904763 + ], + "SC@5": [ + 0.3125, + 12890.080952380951 + ], + "SC@6": [ + 0.325, + 15468.097142857143 + ], + "SC@7": [ + 0.33958333333333335, + 18046.113333333335 + ], + "SC@8": [ + 0.35, + 20624.129523809526 + ], + "AC(0.8)": [ + 0.3458333333333333, + 16319.477916666667 + ], + "AC(0.8)+think@2": [ + 0.5541666666666667, + 15697.92154761905 + ], + "AC(0.8)+think@3": [ + 0.5541666666666667, + 17801.596130952377 + ], + "AC(0.8)+think@4": [ + 0.5354166666666667, + 18999.574940476185 + ], + "AC(0.9)": [ + 0.35, + 19040.267738095237 + ], + "AC(0.9)+think@2": [ + 0.5916666666666667, + 18065.52357142857 + ], + "AC(0.9)+think@3": [ + 0.5729166666666666, + 19318.797767857144 + ], + "AC(0.9)+think@4": [ + 0.5729166666666666, + 21635.043839285718 + ], + "AC(0.95)": [ + 0.35, + 19907.78863095238 + ], + "AC(0.95)+think@2": [ + 0.5916666666666667, + 18065.52357142857 + ], + "AC(0.95)+think@3": [ + 0.5916666666666667, + 20643.53976190476 + ], + "AC(0.95)+think@4": [ + 0.5822916666666667, + 22374.33690476191 + ], + "AC(0.99)": [ + 0.35, + 20433.097738095246 + ], + "AC(0.99)+think@2": [ + 0.5916666666666667, + 18065.52357142857 + ], + "AC(0.99)+think@3": [ + 0.5916666666666667, + 20643.53976190476 + ], + "AC(0.99)+think@4": [ + 0.5916666666666667, + 23221.55595238095 + ], + "always think": [ + 0.5916666666666667, + 12909.491190476188 + ] + } + } + } +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.log b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.log new file mode 100644 index 0000000000000000000000000000000000000000..b0ad4494bfa56a29671001d9b5fb5acfb7696337 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.log @@ -0,0 +1,54 @@ +2026-10-25 14:12:15,499 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4712 +cost model usd: 1 equiv = 4.55e-07 USD; agree2 costs 0.002144 USD per problem + agree2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3562; 0.0005234 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2301 ( -2412; 0.001047 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3451 ( -1261; 0.00157 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4601 ( -111; 0.002094 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5752 ( +1039; 0.002617 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6902 ( +2190; 0.00314 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8052 ( +3340; 0.003664 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9202 ( +4490; 0.004187 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4414 ( -298; 0.002008 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5194 ( +482; 0.002363 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5037 ( +325; 0.002292 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6169 ( +1456; 0.002807 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6443 ( +1730; 0.002931 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7031 ( +2318; 0.003199 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7115 ( +2403; 0.003238 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7916 ( +3204; 0.003602 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8346 ( +3634; 0.003798 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10948 ( +6236; 0.004981 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6347 ( +1635; 0.002888 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1152 +R 16000: acc 0.917 equiv 4048 +R 24000: acc 0.926 equiv 4348 +R 32000: acc 0.933 equiv 4532 +R 48000: acc 0.937 equiv 4752 +R 64000: acc 0.940 equiv 4926 +R 96000: acc 0.941 equiv 5135 +parity: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4693 (-19) | vs agree2 +0.6 [+0.1, +1.1] | R [53811, 35860] | {'stop@2': 3983, 'think:think16k@0': 3576, 'stop@1': 2234, 'stop@3': 232, 'stop@5': 225, 'think:think16k@3': 206, 'stop@4': 126, 'think:think16k@4': 66} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4783 (+71) | vs agree2 +0.5 [+0.0, +1.1] | R [51856, 44619] | {'stop@2': 4097, 'think:think16k@0': 3688, 'stop@1': 2121, 'stop@5': 232, 'stop@4': 146, 'think:think16k@3': 135, 'stop@3': 93, 'stop@8': 72} +split 2: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4755 (+43) | vs agree2 +0.5 [-0.0, +1.0] | R [47340, 42481] | {'stop@2': 3947, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 288, 'think:think16k@2': 169, 'stop@3': 155, 'think:think16k@3': 154, 'stop@8': 86} +split 3: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4602 (-110) | vs agree2 +0.5 [+0.0, +1.1] | R [44144, 43676] | {'think:think16k@0': 3648, 'stop@1': 3492, 'stop@2': 2759, 'stop@5': 267, 'stop@3': 207, 'think:think16k@2': 166, 'think:think16k@3': 145, 'think:think16k@4': 55} +parity at agree2's accuracy: acc 0.932 cost 0.002037 USD vs 0.002144 | cost -5.0% [-7.8, -2.1] | acc +0.2 [-0.3, +0.8] +split 1 at agree2's accuracy: acc 0.929 cost 0.002041 USD vs 0.002144 | cost -4.8% [-7.8, -1.8] | acc -0.1 [-0.6, +0.5] +split 2 at agree2's accuracy: acc 0.931 cost 0.002026 USD vs 0.002144 | cost -5.5% [-8.3, -2.7] | acc +0.1 [-0.4, +0.7] +split 3 at agree2's accuracy: acc 0.931 cost 0.002019 USD vs 0.002144 | cost -5.8% [-8.5, -2.9] | acc +0.1 [-0.4, +0.7] +2026-10-25 14:13:10,657 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '\\alpha^{85}-\\alpha^{-85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:20,932 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '-2' +2026-10-25 14:13:30,974 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '2\\cos(85\\theta)' +2026-10-25 14:13:41,011 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '123' +2026-10-25 14:13:51,047 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' vs '\\alpha^{85}-\\alpha^{-85}' +2026-10-25 14:14:01,391 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: '2' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +2026-10-25 14:14:11,817 judge_selector WARNING: answer comparison timed out after 10s, counted unequal: 'L_{85}' vs 'z^{85}+\\frac{1}{z^{85}}=\\phi^{85}+(\\phi-1)^{85}' +transfer m500: adaptive-SC Bayes acc 0.939 equiv 4947 | agree2 0.924 at 4690 | vs agree2 +1.4 [-0.4, +3.4] | always think 0.938 at 6238 +transfer aime: adaptive-SC Bayes acc 0.533 equiv 17247 | agree2 0.554 at 15698 | vs agree2 -2.1 [-4.2, -0.4] | always think 0.592 at 12909 diff --git a/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.provenance.json b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..390af270999cb3a6d3ddc26a84f8197a07f49474 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/sc8_usd_openrouter_only.provenance.json @@ -0,0 +1,95 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "openrouter_only", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "--match-acc", + "--transfer", + "m500=workdir/selector/pool_m500_k8_chat,workdir/selector/pool_m500_k8_chat/uhead_selector.json,workdir/bayes/m500_tier.json,workdir/bayes/qprior/m500_qprior.json", + "aime=workdir/selector/pool_aime_k8_chat,workdir/selector/pool_aime_k8_chat/uhead_selector.json,workdir/bayes/aime_tier.json,workdir/bayes/qprior/aime_qprior.json", + "--out", + "workdir/bayes/v8/sc8_usd_openrouter_only.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + } + }, + "finished_at": "2026-10-25T14:14:17+04:00", + "wall_secs": 124.0, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..75e02aaa3e70bad826cf8424f36e8454c9893804 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bcf76730b1e7d473f6a6d7ef77ff491829f527c97dc45db6956d7014bb8cc80 +size 327614 diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.json b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.json new file mode 100644 index 0000000000000000000000000000000000000000..3b3d6cb1fa512e0f1fa574cc6830deeb526b456e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.json @@ -0,0 +1,496 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225 + }, + "cost_model": { + "currency": "gpu_s", + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6582624885161, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.3165249770323, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.9747874655477, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.6330499540645, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.29131244258, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.949574931095, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.607837419612, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9197.266099908129, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4412.0116364692885, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5928.330676213297, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5575.824379905499, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.396916034446, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 7354.289774195097, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7941.14717623332, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.669323298678, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 8926.147533635882, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.71692481839, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 12882.837147272214, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 8284.204097318152, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 16000, + "acc": 0.8124540778839089, + "equiv": 1506.2332346595122 + }, + { + "R": 24000, + "acc": 0.9054922850844966, + "equiv": 4730.2022522447505 + }, + { + "R": 32000, + "acc": 0.9287747979426892, + "equiv": 5469.786101770177 + }, + { + "R": 48000, + "acc": 0.9358927259368112, + "equiv": 5813.356175327136 + }, + { + "R": 64000, + "acc": 0.938372520205731, + "equiv": 5983.722530824361 + }, + { + "R": 96000, + "acc": 0.9397961058045555, + "equiv": 6185.602864908272 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9066403379867745, + "equiv": 4780.387792227322, + "R": [ + 31737.957274763416, + 25011.76896809135 + ], + "delta_vs_agree2": [ + -0.023282512858192505, + -0.02902392542248347, + -0.017496326230712712 + ], + "actions": { + "stop@1": 5245, + "stop@2": 2352, + "stop@4": 145, + "think:think16k@0": 1864, + "stop@6": 42, + "stop@5": 179, + "think:think16k@5": 23, + "think:think16k@3": 632, + "think:think16k@6": 10, + "think:think16k@7": 6, + "stop@8": 18, + "stop@7": 34, + "stop@3": 319, + "think:think16k@4": 14, + "think:think16k@8": 5 + } + }, + { + "seed": 1, + "acc": 0.9310249816311535, + "equiv": 5613.162933157068, + "R": [ + 34211.01997690035, + 30133.81637370327 + ], + "delta_vs_agree2": [ + 0.0011021307861866275, + -0.004546289493019839, + 0.006888317413666422 + ], + "actions": { + "stop@2": 2909, + "stop@6": 34, + "stop@5": 118, + "stop@4": 289, + "think:think16k@0": 3688, + "stop@1": 3488, + "think:think16k@5": 72, + "stop@7": 66, + "think:think16k@2": 10, + "stop@3": 62, + "think:think16k@6": 48, + "think:think16k@8": 10, + "think:think16k@3": 70, + "stop@8": 20, + "think:think16k@4": 4 + } + }, + { + "seed": 2, + "acc": 0.9270756796473182, + "equiv": 5450.815983723787, + "R": [ + 34322.76200442399, + 24175.51909619871 + ], + "delta_vs_agree2": [ + -0.002847171197648788, + -0.008496739529757531, + 0.0030779298310066065 + ], + "actions": { + "stop@5": 158, + "stop@4": 134, + "stop@2": 2611, + "stop@1": 3926, + "think:think16k@3": 117, + "think:think16k@2": 42, + "stop@7": 47, + "think:think16k@0": 3600, + "stop@3": 141, + "stop@6": 37, + "think:think16k@8": 6, + "think:think16k@7": 4, + "stop@8": 34, + "think:think16k@4": 13, + "think:think16k@6": 11, + "think:think16k@5": 7 + } + }, + { + "seed": 3, + "acc": 0.9276726671565025, + "equiv": 5433.844433047453, + "R": [ + 29929.841905114903, + 24900.139139552528 + ], + "delta_vs_agree2": [ + -0.0022501836884643643, + -0.008175284717119764, + 0.003950450036737686 + ], + "actions": { + "stop@1": 3963, + "stop@2": 2547, + "think:think16k@3": 94, + "stop@4": 174, + "stop@5": 108, + "stop@6": 55, + "think:think16k@0": 3648, + "stop@3": 181, + "stop@7": 33, + "think:think16k@5": 37, + "stop@8": 18, + "think:think16k@6": 3, + "think:think16k@4": 18, + "think:think16k@8": 4, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9296932402645114, + "equiv": 5526.845547149492, + "R": [ + 31738.04622725488, + 33070.29237209457 + ], + "delta_acc": [ + -0.0002296105804555474, + -0.005923952975753123, + 0.006016945260837613 + ], + "rel_cost": [ + 0.014574388980922596, + -0.016265764902997956, + 0.04722553277173239 + ] + }, + { + "seed": 1, + "acc": 0.9279481998530492, + "equiv": 5501.270490796819, + "R": [ + 24680.402832987907, + 30133.801017737303 + ], + "delta_acc": [ + -0.0019746509919177074, + -0.007763133725202057, + 0.004041146216017634 + ], + "rel_cost": [ + 0.009879523356253932, + -0.022667434990197216, + 0.04355080426891005 + ] + }, + { + "seed": 2, + "acc": 0.9262490815576782, + "equiv": 5406.323231147272, + "R": [ + 31828.864950143754, + 24262.21860421506 + ], + "delta_acc": [ + -0.0036737692872887582, + -0.00936925973548861, + 0.002344324026451126 + ], + "rel_cost": [ + -0.007550140114988803, + -0.039442974188090375, + 0.025170127972970798 + ] + }, + { + "seed": 3, + "acc": 0.9276726671565025, + "equiv": 5433.844433047453, + "R": [ + 29929.86815727641, + 24900.116367032453 + ], + "delta_acc": [ + -0.0022501836884643643, + -0.008175284717119764, + 0.003950450036737686 + ], + "rel_cost": [ + -0.002498016554869073, + -0.03314613177154997, + 0.029405375624232555 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.log b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.log new file mode 100644 index 0000000000000000000000000000000000000000..a231664187b078c999dfa5a88051f9f2d76b677a --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,490 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 5447 +cost model gpu_s: 1 equiv = 0.0006341 GPU-seconds; agree2 costs 3.454 GPU-seconds per problem + agree2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -4298; 0.729 GPU-seconds) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -3148; 1.458 GPU-seconds) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1998; 2.187 GPU-seconds) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4599 ( -849; 2.916 GPU-seconds) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +301; 3.645 GPU-seconds) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +1450; 4.374 GPU-seconds) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8048 ( +2600; 5.103 GPU-seconds) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +3750; 5.832 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -1035; 2.798 GPU-seconds) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5928 ( +481; 3.759 GPU-seconds) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5576 ( +128; 3.536 GPU-seconds) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +718; 3.91 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 7354 ( +1907; 4.663 GPU-seconds) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7941 ( +2494; 5.036 GPU-seconds) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7112 ( +1664; 4.51 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 8926 ( +3479; 5.66 GPU-seconds) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8342 ( +2894; 5.29 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 12883 ( +7435; 8.169 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 8284 ( +2837; 5.253 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.812 equiv 1506 +R 24000: acc 0.905 equiv 4730 +R 32000: acc 0.929 equiv 5470 +R 48000: acc 0.936 equiv 5813 +R 64000: acc 0.938 equiv 5984 +R 96000: acc 0.940 equiv 6186 +parity: adaptive-SC Bayes at agree2's cost acc 0.907 equiv 4780 (-667) | vs agree2 -2.3 [-2.9, -1.7] | R [31738, 25012] | {'stop@1': 5245, 'stop@2': 2352, 'think:think16k@0': 1864, 'think:think16k@3': 632, 'stop@3': 319, 'stop@5': 179, 'stop@4': 145, 'stop@6': 42} +split 1: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 5613 (+166) | vs agree2 +0.1 [-0.5, +0.7] | R [34211, 30134] | {'think:think16k@0': 3688, 'stop@1': 3488, 'stop@2': 2909, 'stop@4': 289, 'stop@5': 118, 'think:think16k@5': 72, 'think:think16k@3': 70, 'stop@7': 66} +split 2: adaptive-SC Bayes at agree2's cost acc 0.927 equiv 5451 (+3) | vs agree2 -0.3 [-0.8, +0.3] | R [34323, 24176] | {'stop@1': 3926, 'think:think16k@0': 3600, 'stop@2': 2611, 'stop@5': 158, 'stop@3': 141, 'stop@4': 134, 'think:think16k@3': 117, 'stop@7': 47} +split 3: adaptive-SC Bayes at agree2's cost acc 0.928 equiv 5434 (-14) | vs agree2 -0.2 [-0.8, +0.4] | R [29930, 24900] | {'stop@1': 3963, 'think:think16k@0': 3648, 'stop@2': 2547, 'stop@3': 181, 'stop@4': 174, 'stop@5': 108, 'think:think16k@3': 94, 'stop@6': 55} +parity at agree2's accuracy: acc 0.930 cost 3.505 GPU-seconds vs 3.454 | cost +1.5% [-1.6, +4.7] | acc -0.0 [-0.6, +0.6] +split 1 at agree2's accuracy: acc 0.928 cost 3.488 GPU-seconds vs 3.454 | cost +1.0% [-2.3, +4.4] | acc -0.2 [-0.8, +0.4] +split 2 at agree2's accuracy: acc 0.926 cost 3.428 GPU-seconds vs 3.454 | cost -0.8% [-3.9, +2.5] | acc -0.4 [-0.9, +0.2] +split 3 at agree2's accuracy: acc 0.928 cost 3.446 GPU-seconds vs 3.454 | cost -0.2% [-3.3, +2.9] | acc -0.2 [-0.8, +0.4] diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..edbbaa59e45a06e8e32be340b6b26f937c92b724 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_gpu_s.provenance.json @@ -0,0 +1,96 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "gpu_s", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scboth_gpu_s.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.1, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..943fff2628aa569e26981f6ec8b4ea935ab3cacf --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b993a929c749fc7fe76c01caf99eb1ca94824eadc900bb00b1ab3f69cfd0140 +size 186053 diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.json b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..b46c639950593dd8b320f4eb19dd15ae368e7cbc --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.json @@ -0,0 +1,517 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734 + }, + "cost_model": { + "currency": "tokens", + "unit": "base tokens", + "per_equiv": 1.0, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1124.9408523144746, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2249.881704628949, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3374.822556943424, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4499.763409257898, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5624.704261572373, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6749.645113886848, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 7874.585966201323, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 8999.526818515797, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4328.9513225569435, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5133.4060525349005, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 4968.986232549595, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6046.053545187362, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6364.151193975019, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 6941.938739897135, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 6970.480804555474, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7812.266908523145, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8168.442964731815, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10841.023512123438, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6341.26010286554, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7909625275532697, + "equiv": 1126.719911829537 + }, + { + "R": 8000, + "acc": 0.7910543717854519, + "equiv": 1127.2466936076414 + }, + { + "R": 16000, + "acc": 0.9371785451873622, + "equiv": 3569.777828802351 + }, + { + "R": 24000, + "acc": 0.9427810433504776, + "equiv": 3768.926644011756 + }, + { + "R": 32000, + "acc": 0.9499908155767818, + "equiv": 3949.579775900073 + }, + { + "R": 48000, + "acc": 0.9543993387215283, + "equiv": 4194.587674504041 + }, + { + "R": 64000, + "acc": 0.9566036002939016, + "equiv": 4316.34729059515 + }, + { + "R": 96000, + "acc": 0.9582108743570904, + "equiv": 4495.031171932403 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9584864070536371, + "equiv": 4625.469893460691, + "R": [ + 175416.9868965556, + 171974.0142994612 + ], + "delta_vs_agree2": [ + 0.028563556208670095, + 0.018780997428361496, + 0.03774912747979426 + ], + "actions": { + "stop@2": 3614, + "stop@3": 2279, + "stop@5": 283, + "think:q32b_think@3": 102, + "think:q32b_think@0": 3576, + "stop@6": 55, + "think:think16k@8": 13, + "think:think16k@7": 33, + "think:q32b_think@6": 6, + "think:q32b_think@5": 6, + "stop@8": 29, + "stop@7": 51, + "think:think16k@3": 95, + "think:think16k@4": 17, + "think:think16k@5": 5, + "think:q32b_think@8": 8, + "think:think16k@6": 8, + "think:q32b_think@4": 14, + "think:q32b_think@1": 522, + "think:think16k@2": 112, + "think:q32b_think@2": 60 + } + }, + { + "seed": 1, + "acc": 0.9583945628214549, + "equiv": 4740.418910727407, + "R": [ + 153946.22121350912, + 130020.67151361922 + ], + "delta_vs_agree2": [ + 0.028471711976487877, + 0.018367698383541513, + 0.03802351212343865 + ], + "actions": { + "stop@8": 87, + "stop@7": 154, + "stop@6": 67, + "stop@5": 323, + "stop@3": 1637, + "think:q32b_think@0": 3688, + "stop@2": 4001, + "think:q32b_think@2": 218, + "think:think16k@4": 82, + "stop@4": 318, + "think:think16k@7": 53, + "think:q32b_think@7": 39, + "think:q32b_think@4": 20, + "think:q32b_think@5": 44, + "think:q32b_think@6": 54, + "think:q32b_think@8": 16, + "think:q32b_think@3": 58, + "think:think16k@6": 12, + "think:think16k@8": 17 + } + }, + { + "seed": 2, + "acc": 0.9569250551065394, + "equiv": 4637.829932035268, + "R": [ + 137175.13058426927, + 119496.99348268835 + ], + "delta_vs_agree2": [ + 0.027002204261572373, + 0.017264419544452608, + 0.03646445628214547 + ], + "actions": { + "think:q32b_think@2": 395, + "stop@8": 90, + "stop@7": 47, + "stop@6": 64, + "stop@5": 292, + "stop@4": 629, + "stop@3": 1331, + "stop@2": 4106, + "think:think16k@3": 16, + "think:think16k@4": 8, + "think:q32b_think@0": 3600, + "think:q32b_think@4": 95, + "think:think16k@7": 2, + "think:q32b_think@7": 11, + "think:q32b_think@5": 26, + "think:q32b_think@6": 30, + "think:q32b_think@8": 8, + "think:q32b_think@3": 43, + "think:think16k@8": 16, + "think:think16k@6": 24, + "think:think16k@5": 1, + "think:think16k@2": 54 + } + }, + { + "seed": 3, + "acc": 0.9583027185892726, + "equiv": 4773.641256429096, + "R": [ + 167008.99393426898, + 165785.6583409342 + ], + "delta_vs_agree2": [ + 0.02837986774430566, + 0.018047391623806024, + 0.03871234386480529 + ], + "actions": { + "stop@2": 2633, + "stop@3": 1632, + "think:q32b_think@1": 399, + "stop@5": 216, + "think:q32b_think@0": 5448, + "think:q32b_think@3": 4, + "stop@6": 41, + "think:think16k@3": 6, + "think:think16k@4": 18, + "stop@7": 73, + "think:think16k@8": 17, + "think:think16k@6": 22, + "think:q32b_think@2": 99, + "stop@8": 18, + "think:q32b_think@4": 10, + "think:think16k@7": 9, + "think:q32b_think@5": 1, + "think:q32b_think@8": 1, + "stop@4": 239, + "think:q32b_think@6": 2 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9315760470242469, + "equiv": 3503.206631153563, + "R": [ + 11423.401733790855, + 10518.13922265171 + ], + "delta_acc": [ + 0.0016531961792799412, + -0.008818194342395298, + 0.012400119397501831 + ], + "rel_cost": [ + -0.24817845377310907, + -0.28184686536233694, + -0.21457295674894686 + ] + }, + { + "seed": 1, + "acc": 0.9357090374724467, + "equiv": 3590.1690852314473, + "R": [ + 16363.0063018621, + 10384.692857801403 + ], + "delta_acc": [ + 0.005786186627479794, + -0.004592211609110948, + 0.016349421381337246 + ], + "rel_cost": [ + -0.2295154819383616, + -0.2626906064253444, + -0.19661994284176215 + ] + }, + { + "seed": 2, + "acc": 0.9327700220426157, + "equiv": 3531.7552534900806, + "R": [ + 10636.670927247456, + 11919.236671515018 + ], + "delta_acc": [ + 0.002847171197648788, + -0.007440530859662013, + 0.01317964731814842 + ], + "rel_cost": [ + -0.24205164720771377, + -0.27471063315128796, + -0.2090280731623695 + ] + }, + { + "seed": 3, + "acc": 0.9351579720793534, + "equiv": 3551.275404114622, + "R": [ + 10864.127712879404, + 10861.721846413127 + ], + "delta_acc": [ + 0.00523512123438648, + -0.004777048126377663, + 0.015660589639970603 + ], + "rel_cost": [ + -0.23786243675845992, + -0.2699880503506846, + -0.20568122642560924 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.log b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.log new file mode 100644 index 0000000000000000000000000000000000000000..4b931f563088834c938b6426f5aefcb16d09fa3e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,487 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4660 +cost model tokens: 1 equiv = 1 base tokens; agree2 costs 4660 base tokens per problem + agree2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1125 ( -3535; 1125 base tokens) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2250 ( -2410; 2250 base tokens) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3375 ( -1285; 3375 base tokens) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4500 ( -160; 4500 base tokens) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5625 ( +965; 5625 base tokens) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6750 ( +2090; 6750 base tokens) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 7875 ( +3215; 7875 base tokens) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9000 ( +4340; 9000 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4329 ( -331; 4329 base tokens) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5133 ( +474; 5133 base tokens) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 4969 ( +309; 4969 base tokens) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6046 ( +1386; 6046 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6364 ( +1705; 6364 base tokens) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 6942 ( +2282; 6942 base tokens) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 6970 ( +2311; 6970 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7812 ( +3153; 7812 base tokens) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8168 ( +3509; 8168 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10841 ( +6181; 1.084e+04 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6341 ( +1682; 6341 base tokens) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1127 +R 8000: acc 0.791 equiv 1127 +R 16000: acc 0.937 equiv 3570 +R 24000: acc 0.943 equiv 3769 +R 32000: acc 0.950 equiv 3950 +R 48000: acc 0.954 equiv 4195 +R 64000: acc 0.957 equiv 4316 +R 96000: acc 0.958 equiv 4495 +parity: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 4625 (-34) | vs agree2 +2.9 [+1.9, +3.8] | R [175417, 171974] | {'stop@2': 3614, 'think:q32b_think@0': 3576, 'stop@3': 2279, 'think:q32b_think@1': 522, 'stop@5': 283, 'think:think16k@2': 112, 'think:q32b_think@3': 102, 'think:think16k@3': 95} +split 1: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 4740 (+81) | vs agree2 +2.8 [+1.8, +3.8] | R [153946, 130021] | {'stop@2': 4001, 'think:q32b_think@0': 3688, 'stop@3': 1637, 'stop@5': 323, 'stop@4': 318, 'think:q32b_think@2': 218, 'stop@7': 154, 'stop@8': 87} +split 2: adaptive-SC Bayes at agree2's cost acc 0.957 equiv 4638 (-22) | vs agree2 +2.7 [+1.7, +3.6] | R [137175, 119497] | {'stop@2': 4106, 'think:q32b_think@0': 3600, 'stop@3': 1331, 'stop@4': 629, 'think:q32b_think@2': 395, 'stop@5': 292, 'think:q32b_think@4': 95, 'stop@8': 90} +split 3: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 4774 (+114) | vs agree2 +2.8 [+1.8, +3.9] | R [167009, 165786] | {'think:q32b_think@0': 5448, 'stop@2': 2633, 'stop@3': 1632, 'think:q32b_think@1': 399, 'stop@4': 239, 'stop@5': 216, 'think:q32b_think@2': 99, 'stop@7': 73} +parity at agree2's accuracy: acc 0.932 cost 3503 base tokens vs 4660 | cost -24.8% [-28.2, -21.5] | acc +0.2 [-0.9, +1.2] +split 1 at agree2's accuracy: acc 0.936 cost 3590 base tokens vs 4660 | cost -23.0% [-26.3, -19.7] | acc +0.6 [-0.5, +1.6] +split 2 at agree2's accuracy: acc 0.933 cost 3532 base tokens vs 4660 | cost -24.2% [-27.5, -20.9] | acc +0.3 [-0.7, +1.3] +split 3 at agree2's accuracy: acc 0.935 cost 3551 base tokens vs 4660 | cost -23.8% [-27.0, -20.6] | acc +0.5 [-0.5, +1.6] diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..880dcdf89deda3918064803a523809d82867c460 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_tokens.provenance.json @@ -0,0 +1,96 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "tokens", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json", + "q32b_think=workdir/bayes/q32b_think_tier.json", + "--match-acc", + "--out", + "workdir/bayes/v8/scboth_tokens.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.4, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..db070b5f247a711539a43cd201ce96ac94f8cf88 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a270133b14666a378f4886456f80f04033dd4aaca8e6bb06e8007f0c05003a3 +size 183097 diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.json new file mode 100644 index 0000000000000000000000000000000000000000..8256de252060ae5919e5cf3bd54b4c7e0ba2928a --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.json @@ -0,0 +1,517 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "cheapest", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1151.652736958119 + }, + { + "R": 16000, + "acc": 0.9172483468038207, + "equiv": 4055.122864621602 + }, + { + "R": 24000, + "acc": 0.9347905951506246, + "equiv": 4657.303788574577 + }, + { + "R": 32000, + "acc": 0.9497152828802351, + "equiv": 5131.996284900808 + }, + { + "R": 48000, + "acc": 0.953848273328435, + "equiv": 5342.718125459221 + }, + { + "R": 64000, + "acc": 0.9557770022042615, + "equiv": 5513.992454996327 + }, + { + "R": 96000, + "acc": 0.9571546656869948, + "equiv": 5710.908096069066 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9386480529022777, + "equiv": 4825.261526451139, + "R": [ + 26406.83650444677, + 22719.972726155036 + ], + "delta_vs_agree2": [ + 0.008725202057310802, + -0.001239897134459956, + 0.019289584864070525 + ], + "actions": { + "stop@1": 5584, + "stop@2": 1212, + "stop@4": 59, + "think:q32b_think@0": 3576, + "stop@3": 70, + "stop@6": 12, + "think:think16k@3": 37, + "stop@5": 59, + "think:think16k@4": 21, + "think:q32b_think@3": 40, + "think:think16k@2": 49, + "think:q32b_think@5": 2, + "stop@8": 2, + "think:think16k@6": 2, + "think:think16k@5": 3, + "think:q32b_think@2": 160 + } + }, + { + "seed": 1, + "acc": 0.9347905951506246, + "equiv": 5018.314433321088, + "R": [ + 52930.840821866186, + 16864.897675800123 + ], + "delta_vs_agree2": [ + 0.0048677443056576045, + -0.003402828802351212, + 0.01345747612049962 + ], + "actions": { + "stop@8": 66, + "stop@7": 32, + "stop@6": 35, + "stop@4": 133, + "stop@3": 153, + "stop@2": 3591, + "think:think16k@0": 1560, + "stop@5": 115, + "think:think16k@3": 85, + "think:think16k@4": 35, + "think:think16k@2": 10, + "think:think16k@5": 14, + "think:q32b_think@4": 50, + "think:q32b_think@7": 15, + "think:q32b_think@6": 22, + "think:q32b_think@5": 7, + "think:think16k@6": 8, + "think:think16k@7": 7, + "think:think16k@8": 1, + "think:q32b_think@8": 1, + "stop@1": 2726, + "think:q32b_think@0": 2128, + "think:q32b_think@2": 94 + } + }, + { + "seed": 2, + "acc": 0.9305657604702424, + "equiv": 4599.510805473917, + "R": [ + 28666.976383164332, + 36234.06037995943 + ], + "delta_vs_agree2": [ + 0.0006429096252755327, + -0.007302764511388685, + 0.008908890521675239 + ], + "actions": { + "stop@1": 4731, + "stop@3": 150, + "stop@2": 1939, + "think:think16k@3": 42, + "think:think16k@2": 42, + "think:q32b_think@0": 1616, + "stop@5": 50, + "stop@4": 98, + "stop@7": 19, + "think:q32b_think@3": 57, + "stop@6": 17, + "think:think16k@7": 4, + "stop@8": 10, + "think:q32b_think@5": 5, + "think:think16k@8": 2, + "think:q32b_think@7": 1, + "think:q32b_think@8": 1, + "think:q32b_think@6": 2, + "think:q32b_think@2": 110, + "think:think16k@0": 1984, + "think:think16k@5": 4, + "think:think16k@4": 3, + "think:think16k@6": 1 + } + }, + { + "seed": 3, + "acc": 0.9363519470977223, + "equiv": 4666.87691495224, + "R": [ + 21357.26622106588, + 35450.41050337804 + ], + "delta_vs_agree2": [ + 0.006429096252755327, + -0.0017921105804555473, + 0.015154298310066127 + ], + "actions": { + "stop@1": 4841, + "stop@2": 1745, + "think:q32b_think@2": 237, + "stop@3": 134, + "think:think16k@0": 1664, + "stop@4": 12, + "think:think16k@3": 55, + "stop@8": 9, + "think:q32b_think@4": 1, + "stop@6": 20, + "stop@5": 123, + "think:q32b_think@0": 1984, + "think:think16k@2": 25, + "stop@7": 11, + "think:think16k@5": 12, + "think:think16k@6": 6, + "think:think16k@7": 5, + "think:think16k@8": 1, + "think:q32b_think@5": 2, + "think:think16k@4": 1 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9386480529022777, + "equiv": 4825.261526451139, + "R": [ + 26406.82978407768, + 22719.943792587368 + ], + "delta_acc": [ + 0.008725202057310802, + -0.001239897134459956, + 0.019289584864070525 + ], + "rel_cost": [ + 0.024352262884880075, + -0.012057640162018926, + 0.06397871541077248 + ] + }, + { + "seed": 1, + "acc": 0.9283614988978692, + "equiv": 4677.113087803086, + "R": [ + 24028.359372219857, + 16864.868338774082 + ], + "delta_acc": [ + -0.0015613519470977222, + -0.010152231814842027, + 0.007255694342395298 + ], + "rel_cost": [ + -0.007098092197450084, + -0.04199130367198117, + 0.02780604544404557 + ] + }, + { + "seed": 2, + "acc": 0.9284074210139603, + "equiv": 4488.920159808964, + "R": [ + 18611.932132232734, + 22065.52260411527 + ], + "delta_acc": [ + -0.0015154298310066127, + -0.009597722263041881, + 0.006751699118295365 + ], + "rel_cost": [ + -0.047049471121255695, + -0.08034524749062692, + -0.01435037379024317 + ] + }, + { + "seed": 3, + "acc": 0.9310709037472447, + "equiv": 4570.490691587068, + "R": [ + 21357.313705120658, + 24058.502781455634 + ], + "delta_acc": [ + 0.001148052902277737, + -0.007073153930933137, + 0.009873254959588539 + ], + "rel_cost": [ + -0.029732905303303386, + -0.0626666385414301, + 0.0034243698253159467 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.log b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.log new file mode 100644 index 0000000000000000000000000000000000000000..1fa047edaed2b80f03fb3a9058d62e4cca30bae0 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,477 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1152 +R 16000: acc 0.917 equiv 4055 +R 24000: acc 0.935 equiv 4657 +R 32000: acc 0.950 equiv 5132 +R 48000: acc 0.954 equiv 5343 +R 64000: acc 0.956 equiv 5514 +R 96000: acc 0.957 equiv 5711 +parity: adaptive-SC Bayes at agree2's cost acc 0.939 equiv 4825 (+115) | vs agree2 +0.9 [-0.1, +1.9] | R [26407, 22720] | {'stop@1': 5584, 'think:q32b_think@0': 3576, 'stop@2': 1212, 'think:q32b_think@2': 160, 'stop@3': 70, 'stop@4': 59, 'stop@5': 59, 'think:think16k@2': 49} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 5018 (+308) | vs agree2 +0.5 [-0.3, +1.3] | R [52931, 16865] | {'stop@2': 3591, 'stop@1': 2726, 'think:q32b_think@0': 2128, 'think:think16k@0': 1560, 'stop@3': 153, 'stop@4': 133, 'stop@5': 115, 'think:q32b_think@2': 94} +split 2: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 4600 (-111) | vs agree2 +0.1 [-0.7, +0.9] | R [28667, 36234] | {'stop@1': 4731, 'think:think16k@0': 1984, 'stop@2': 1939, 'think:q32b_think@0': 1616, 'stop@3': 150, 'think:q32b_think@2': 110, 'stop@4': 98, 'think:q32b_think@3': 57} +split 3: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4667 (-44) | vs agree2 +0.6 [-0.2, +1.5] | R [21357, 35450] | {'stop@1': 4841, 'think:q32b_think@0': 1984, 'stop@2': 1745, 'think:think16k@0': 1664, 'think:q32b_think@2': 237, 'stop@3': 134, 'stop@5': 123, 'think:think16k@3': 55} +parity at agree2's accuracy: acc 0.939 cost 0.0009651 USD vs 0.0009421 | cost +2.4% [-1.2, +6.4] | acc +0.9 [-0.1, +1.9] +split 1 at agree2's accuracy: acc 0.928 cost 0.0009354 USD vs 0.0009421 | cost -0.7% [-4.2, +2.8] | acc -0.2 [-1.0, +0.7] +split 2 at agree2's accuracy: acc 0.928 cost 0.0008978 USD vs 0.0009421 | cost -4.7% [-8.0, -1.4] | acc -0.2 [-1.0, +0.7] +split 3 at agree2's accuracy: acc 0.931 cost 0.0009141 USD vs 0.0009421 | cost -3.0% [-6.3, +0.3] | acc +0.1 [-0.7, +1.0] diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..90752f72526162f418f588cab6bb4bb04088e0c5 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_cheapest.provenance.json @@ -0,0 +1,98 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "cheapest", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scboth_usd_cheapest.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:56+04:00", + "wall_secs": 43.0, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..c573cde6bea61339f91792fd3d3feff13643cff6 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40516fcb4e96d30a73ecaacc02899e75e1af6618ed5d33aecc368295c44f8b43 +size 181636 diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.json new file mode 100644 index 0000000000000000000000000000000000000000..6890db54a546d1a858620f514180aea2677c29cf --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.json @@ -0,0 +1,499 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "dear_32b", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.5956557678178 + }, + { + "R": 8000, + "acc": 0.7908706833210875, + "equiv": 1151.652736958119 + }, + { + "R": 16000, + "acc": 0.9168350477590007, + "equiv": 4046.822556943424 + }, + { + "R": 24000, + "acc": 0.926203159441587, + "equiv": 4346.849260653931 + }, + { + "R": 32000, + "acc": 0.9332292432035268, + "equiv": 4530.6730574944895 + }, + { + "R": 48000, + "acc": 0.936627479794269, + "equiv": 4750.71252755327 + }, + { + "R": 64000, + "acc": 0.9396583394562822, + "equiv": 4924.973838170463 + }, + { + "R": 96000, + "acc": 0.9409900808229243, + "equiv": 5232.961576965466 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9357549595885378, + "equiv": 4665.914447097723, + "R": [ + 53761.60815882869, + 35840.683944266966 + ], + "delta_vs_agree2": [ + 0.0058321087435709035, + 0.0008265980896399706, + 0.011296840558412931 + ], + "actions": { + "stop@1": 2914, + "stop@2": 3300, + "stop@4": 128, + "stop@5": 229, + "stop@3": 232, + "think:think16k@0": 3576, + "stop@6": 35, + "think:think16k@3": 204, + "think:think16k@8": 6, + "think:think16k@6": 10, + "think:think16k@4": 72, + "stop@8": 20, + "think:think16k@7": 8, + "think:think16k@2": 49, + "stop@7": 61, + "think:think16k@5": 44 + } + }, + { + "seed": 1, + "acc": 0.9350661278471712, + "equiv": 4767.403655400441, + "R": [ + 51814.74721102044, + 44596.64305701744 + ], + "delta_vs_agree2": [ + 0.005143277002204262, + 0.00013776634827332844, + 0.010518460690668613 + ], + "actions": { + "stop@8": 43, + "stop@7": 44, + "stop@6": 40, + "stop@4": 191, + "stop@3": 93, + "stop@2": 4097, + "think:think16k@0": 3688, + "stop@1": 2121, + "stop@5": 224, + "think:think16k@3": 135, + "think:think16k@4": 62, + "think:think16k@2": 10, + "think:think16k@5": 55, + "think:think16k@7": 22, + "think:think16k@6": 53, + "think:think16k@8": 10 + } + }, + { + "seed": 2, + "acc": 0.9344232182218957, + "equiv": 4708.9382117928, + "R": [ + 47298.906892707404, + 42459.620096407016 + ], + "delta_vs_agree2": [ + 0.004500367376928729, + -0.0005051432770022043, + 0.009919177075679648 + ], + "actions": { + "stop@5": 271, + "stop@8": 83, + "stop@7": 30, + "stop@6": 34, + "stop@4": 30, + "stop@3": 155, + "stop@2": 3461, + "stop@1": 2778, + "think:think16k@3": 154, + "think:think16k@2": 169, + "think:think16k@4": 50, + "think:think16k@0": 3600, + "think:think16k@8": 14, + "think:think16k@7": 12, + "think:think16k@5": 22, + "think:think16k@6": 25 + } + }, + { + "seed": 3, + "acc": 0.9362601028655401, + "equiv": 4689.776979243204, + "R": [ + 44122.35950080606, + 43636.44372207239 + ], + "delta_vs_agree2": [ + 0.006337252020573108, + 0.0013765154298310068, + 0.011757209772226298 + ], + "actions": { + "stop@1": 2454, + "stop@2": 3669, + "stop@3": 279, + "stop@6": 47, + "think:think16k@3": 145, + "stop@5": 297, + "think:think16k@0": 3648, + "think:think16k@4": 59, + "stop@7": 28, + "think:think16k@5": 35, + "think:think16k@8": 4, + "think:think16k@2": 166, + "think:think16k@6": 16, + "stop@8": 8, + "stop@4": 28, + "think:think16k@7": 5 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9318056576047025, + "equiv": 4474.977612968406, + "R": [ + 30117.061370323576, + 27672.9869610281 + ], + "delta_acc": [ + 0.0018828067597354886, + -0.0032145481263776633, + 0.007578297207935335 + ], + "rel_cost": [ + -0.05000932714734109, + -0.07769058467004979, + -0.021280741992484378 + ] + }, + { + "seed": 1, + "acc": 0.9291880969875091, + "equiv": 4485.228577332843, + "R": [ + 24061.26226762234, + 28313.979910682214 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.0061994856722997795, + 0.004913666421748714 + ], + "rel_cost": [ + -0.04783315524745505, + -0.07812418369181799, + -0.017764597058704746 + ] + }, + { + "seed": 2, + "acc": 0.9310249816311535, + "equiv": 4452.599467303453, + "R": [ + 31420.259330627625, + 22070.104606704397 + ], + "delta_acc": [ + 0.0011021307861866275, + -0.003996372152828802, + 0.006613932770022036 + ], + "rel_cost": [ + -0.054759972065839574, + -0.08278413995488167, + -0.026639383846745635 + ] + }, + { + "seed": 3, + "acc": 0.9310709037472447, + "equiv": 4437.428430382072, + "R": [ + 28652.598441905946, + 24064.307585194245 + ], + "delta_acc": [ + 0.001148052902277737, + -0.004087068332108743, + 0.006842395297575312 + ], + "rel_cost": [ + -0.05798062361302248, + -0.08510425518256229, + -0.02927755089924474 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.log b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.log new file mode 100644 index 0000000000000000000000000000000000000000..373580f9337553e385955253a6b4ce35ff203c34 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,493 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1152 +R 16000: acc 0.917 equiv 4047 +R 24000: acc 0.926 equiv 4347 +R 32000: acc 0.933 equiv 4531 +R 48000: acc 0.937 equiv 4751 +R 64000: acc 0.940 equiv 4925 +R 96000: acc 0.941 equiv 5233 +parity: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4666 (-45) | vs agree2 +0.6 [+0.1, +1.1] | R [53762, 35841] | {'think:think16k@0': 3576, 'stop@2': 3300, 'stop@1': 2914, 'stop@3': 232, 'stop@5': 229, 'think:think16k@3': 204, 'stop@4': 128, 'think:think16k@4': 72} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4767 (+57) | vs agree2 +0.5 [+0.0, +1.1] | R [51815, 44597] | {'stop@2': 4097, 'think:think16k@0': 3688, 'stop@1': 2121, 'stop@5': 224, 'stop@4': 191, 'think:think16k@3': 135, 'stop@3': 93, 'think:think16k@4': 62} +split 2: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4709 (-2) | vs agree2 +0.5 [-0.1, +1.0] | R [47299, 42460] | {'think:think16k@0': 3600, 'stop@2': 3461, 'stop@1': 2778, 'stop@5': 271, 'think:think16k@2': 169, 'stop@3': 155, 'think:think16k@3': 154, 'stop@8': 83} +split 3: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4690 (-21) | vs agree2 +0.6 [+0.1, +1.2] | R [44122, 43636] | {'stop@2': 3669, 'think:think16k@0': 3648, 'stop@1': 2454, 'stop@5': 297, 'stop@3': 279, 'think:think16k@2': 166, 'think:think16k@3': 145, 'think:think16k@4': 59} +parity at agree2's accuracy: acc 0.932 cost 0.000895 USD vs 0.0009421 | cost -5.0% [-7.8, -2.1] | acc +0.2 [-0.3, +0.8] +split 1 at agree2's accuracy: acc 0.929 cost 0.000897 USD vs 0.0009421 | cost -4.8% [-7.8, -1.8] | acc -0.1 [-0.6, +0.5] +split 2 at agree2's accuracy: acc 0.931 cost 0.0008905 USD vs 0.0009421 | cost -5.5% [-8.3, -2.7] | acc +0.1 [-0.4, +0.7] +split 3 at agree2's accuracy: acc 0.931 cost 0.0008875 USD vs 0.0009421 | cost -5.8% [-8.5, -2.9] | acc +0.1 [-0.4, +0.7] diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..017b75548433f43879d95c0d0a4a051a078dedc9 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_dear_32b.provenance.json @@ -0,0 +1,98 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "dear_32b", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scboth_usd_dear_32b.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.0, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..c2fbebaec5ed284e8e01eadaeac9bff85c7f3a71 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c70445ffa268bb17131c33a44d2b647ad2d2d7881addb626d9badf47d0ce26f5 +size 232273 diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.json new file mode 100644 index 0000000000000000000000000000000000000000..be43b84f24bd8f3f4c1ebfd8702ab2be2b1690a1 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.json @@ -0,0 +1,524 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "scenario": "openrouter_only", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1150.3112732234702, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2300.6225464469403, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3450.9338196704107, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4601.245092893881, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5751.556366117351, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6901.867639340821, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8052.178912564291, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9202.490185787761, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4414.206011861026, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5194.155375511704, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5037.402507347539, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6168.549855673349, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6442.532221580771, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7030.53273328435, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7115.399391203946, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7916.276974650992, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8346.294659913927, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10948.143067072531, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.89797417865, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.8467119764878765, + "equiv": 1677.7417610677344 + }, + { + "R": 8000, + "acc": 0.9099008082292432, + "equiv": 2190.841744717443 + }, + { + "R": 16000, + "acc": 0.9398420279206466, + "equiv": 2525.680790021881 + }, + { + "R": 24000, + "acc": 0.945077149155033, + "equiv": 2701.0134649700044 + }, + { + "R": 32000, + "acc": 0.9544452608376194, + "equiv": 2880.1967099579333 + }, + { + "R": 48000, + "acc": 0.9570168993387216, + "equiv": 3008.6927386537045 + }, + { + "R": 64000, + "acc": 0.9589456282145481, + "equiv": 3138.332467037005 + }, + { + "R": 96000, + "acc": 0.9583486407053637, + "equiv": 3323.760166147225 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9576138868479059, + "equiv": 5618.6873994759835, + "R": [ + 499998.353065514, + 232207.81824516953 + ], + "delta_vs_agree2": [ + 0.027691036002939016, + 0.018503168626010286, + 0.0369684515062454 + ], + "actions": { + "stop@4": 2686, + "stop@5": 315, + "stop@8": 117, + "stop@6": 153, + "think:q32b_think@0": 1864, + "think:think16k@8": 26, + "think:think16k@7": 52, + "think:q32b_think@6": 74, + "stop@3": 1880, + "think:think16k@3": 6, + "stop@7": 34, + "think:q32b_think@8": 537, + "think:think16k@6": 22, + "think:think16k@5": 14, + "think:think16k@4": 25, + "think:q32b_think@7": 53, + "think:q32b_think@1": 1091, + "think:think16k@1": 844, + "stop@2": 558, + "think:q32b_think@2": 216, + "think:q32b_think@4": 74, + "think:q32b_think@3": 104, + "think:q32b_think@5": 143 + } + }, + { + "seed": 1, + "acc": 0.9583027185892726, + "equiv": 4559.63655440812, + "R": [ + 377564.9936511542, + 499998.353065514 + ], + "delta_vs_agree2": [ + 0.02837986774430566, + 0.01772593681116826, + 0.03844025532696545 + ], + "actions": { + "think:q32b_think@0": 3984, + "think:q32b_think@8": 343, + "think:q32b_think@1": 920, + "stop@2": 2751, + "stop@5": 192, + "stop@3": 440, + "think:q32b_think@3": 132, + "think:q32b_think@2": 256, + "think:q32b_think@4": 148, + "think:q32b_think@5": 119, + "think:q32b_think@6": 57, + "stop@6": 18, + "think:q32b_think@7": 42, + "stop@4": 1297, + "stop@7": 42, + "think:think16k@5": 3, + "stop@8": 78, + "think:think16k@7": 10, + "think:think16k@8": 22, + "think:think16k@4": 1, + "think:think16k@3": 1, + "think:think16k@2": 32 + } + }, + { + "seed": 2, + "acc": 0.9597722263041881, + "equiv": 4079.052407933727, + "R": [ + 499998.353065514, + 429561.0110198085 + ], + "delta_vs_agree2": [ + 0.029849375459221162, + 0.019792432035268186, + 0.03963078618662748 + ], + "actions": { + "think:q32b_think@2": 329, + "think:think16k@8": 30, + "think:think16k@7": 13, + "think:think16k@6": 31, + "think:think16k@5": 31, + "stop@4": 3121, + "think:think16k@4": 75, + "think:think16k@3": 108, + "stop@2": 1365, + "think:q32b_think@5": 51, + "think:q32b_think@0": 3600, + "stop@6": 117, + "stop@5": 212, + "stop@3": 290, + "think:q32b_think@4": 81, + "think:q32b_think@7": 30, + "stop@8": 90, + "stop@7": 22, + "think:q32b_think@3": 94, + "think:q32b_think@8": 33, + "think:q32b_think@6": 10, + "think:q32b_think@1": 1118, + "think:think16k@2": 37 + } + }, + { + "seed": 3, + "acc": 0.9580731080088171, + "equiv": 3605.338127972321, + "R": [ + 476352.67793622956, + 499998.353065514 + ], + "delta_vs_agree2": [ + 0.02815025716385011, + 0.017450404114621603, + 0.0387582659808964 + ], + "actions": { + "stop@3": 1803, + "think:q32b_think@1": 870, + "stop@4": 1177, + "stop@5": 194, + "think:q32b_think@0": 5448, + "think:q32b_think@3": 19, + "think:think16k@5": 98, + "stop@8": 19, + "think:think16k@3": 7, + "think:think16k@4": 26, + "stop@7": 28, + "think:think16k@7": 30, + "think:think16k@8": 21, + "think:q32b_think@4": 17, + "stop@2": 1052, + "stop@6": 24, + "think:think16k@6": 2, + "think:q32b_think@8": 6, + "think:q32b_think@5": 1, + "think:q32b_think@7": 1, + "think:q32b_think@2": 45 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9323108008817046, + "equiv": 2429.8270902132404, + "R": [ + 9200.130649776325, + 8660.485855178304 + ], + "delta_acc": [ + 0.002387950036737693, + -0.007945674136664218, + 0.013088951138868474 + ], + "rel_cost": [ + -0.4843524611018131, + -0.5214862717242087, + -0.44859757004934486 + ] + }, + { + "seed": 1, + "acc": 0.9335966201322556, + "equiv": 2456.498128396218, + "R": [ + 9937.983745778412, + 8446.1985364643 + ], + "delta_acc": [ + 0.0036737692872887582, + -0.006843543350477589, + 0.014099237692872883 + ], + "rel_cost": [ + -0.47869244716324716, + -0.5166862914494847, + -0.4433462108907839 + ] + }, + { + "seed": 2, + "acc": 0.9356171932402645, + "equiv": 2469.2057506600677, + "R": [ + 9155.23145879943, + 9744.223353889529 + ], + "delta_acc": [ + 0.005694342395297576, + -0.004456741366642174, + 0.01593497428361499 + ], + "rel_cost": [ + -0.4759956897799773, + -0.5124283478500522, + -0.4401570704756052 + ] + }, + { + "seed": 3, + "acc": 0.9346987509184423, + "equiv": 2442.8669245706533, + "R": [ + 8731.090330941195, + 8679.808070658055 + ], + "delta_acc": [ + 0.004775900073475386, + -0.005418809698750918, + 0.01524729059515062 + ], + "rel_cost": [ + -0.48158520308533836, + -0.5179828783177912, + -0.445889791447233 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.log b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.log new file mode 100644 index 0000000000000000000000000000000000000000..e738333455232049879ffbc688e40a15f5e80eb5 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,486 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4712 +cost model usd: 1 equiv = 4.55e-07 USD; agree2 costs 0.002144 USD per problem + agree2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3562; 0.0005234 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2301 ( -2412; 0.001047 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3451 ( -1261; 0.00157 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4601 ( -111; 0.002094 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5752 ( +1039; 0.002617 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6902 ( +2190; 0.00314 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8052 ( +3340; 0.003664 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9202 ( +4490; 0.004187 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4414 ( -298; 0.002008 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5194 ( +482; 0.002363 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5037 ( +325; 0.002292 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6169 ( +1456; 0.002807 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6443 ( +1730; 0.002931 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7031 ( +2318; 0.003199 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7115 ( +2403; 0.003238 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7916 ( +3204; 0.003602 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8346 ( +3634; 0.003798 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10948 ( +6236; 0.004981 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6347 ( +1635; 0.002888 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.847 equiv 1678 +R 8000: acc 0.910 equiv 2191 +R 16000: acc 0.940 equiv 2526 +R 24000: acc 0.945 equiv 2701 +R 32000: acc 0.954 equiv 2880 +R 48000: acc 0.957 equiv 3009 +R 64000: acc 0.959 equiv 3138 +R 96000: acc 0.958 equiv 3324 +parity: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 5619 (+907) | vs agree2 +2.8 [+1.9, +3.7] | R [499998, 232208] | {'stop@4': 2686, 'stop@3': 1880, 'think:q32b_think@0': 1864, 'think:q32b_think@1': 1091, 'think:think16k@1': 844, 'stop@2': 558, 'think:q32b_think@8': 537, 'stop@5': 315} +split 1: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 4560 (-153) | vs agree2 +2.8 [+1.8, +3.8] | R [377565, 499998] | {'think:q32b_think@0': 3984, 'stop@2': 2751, 'stop@4': 1297, 'think:q32b_think@1': 920, 'stop@3': 440, 'think:q32b_think@8': 343, 'think:q32b_think@2': 256, 'stop@5': 192} +split 2: adaptive-SC Bayes at agree2's cost acc 0.960 equiv 4079 (-633) | vs agree2 +3.0 [+2.0, +4.0] | R [499998, 429561] | {'think:q32b_think@0': 3600, 'stop@4': 3121, 'stop@2': 1365, 'think:q32b_think@1': 1118, 'think:q32b_think@2': 329, 'stop@3': 290, 'stop@5': 212, 'stop@6': 117} +split 3: adaptive-SC Bayes at agree2's cost acc 0.958 equiv 3605 (-1107) | vs agree2 +2.8 [+1.7, +3.9] | R [476353, 499998] | {'think:q32b_think@0': 5448, 'stop@3': 1803, 'stop@4': 1177, 'stop@2': 1052, 'think:q32b_think@1': 870, 'stop@5': 194, 'think:think16k@5': 98, 'think:q32b_think@2': 45} +parity at agree2's accuracy: acc 0.932 cost 0.001106 USD vs 0.002144 | cost -48.4% [-52.1, -44.9] | acc +0.2 [-0.8, +1.3] +split 1 at agree2's accuracy: acc 0.934 cost 0.001118 USD vs 0.002144 | cost -47.9% [-51.7, -44.3] | acc +0.4 [-0.7, +1.4] +split 2 at agree2's accuracy: acc 0.936 cost 0.001123 USD vs 0.002144 | cost -47.6% [-51.2, -44.0] | acc +0.6 [-0.4, +1.6] +split 3 at agree2's accuracy: acc 0.935 cost 0.001112 USD vs 0.002144 | cost -48.2% [-51.8, -44.6] | acc +0.5 [-0.5, +1.5] diff --git a/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..a8222d3efd62d1cee1325567716922523590f40d --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scboth_usd_openrouter_only.provenance.json @@ -0,0 +1,98 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "openrouter_only", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "q32b_think=workdir/bayes/q32b_think_tier.json@qwen3-32b-fp8:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scboth_usd_openrouter_only.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:q32b_think": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/q32b_think_tier.json", + "bytes": 26525, + "sha256": "4eef554b7fa4edff6cf11b63aadb44267d211bc25285eb260dcc1b45d2cf3a1f" + } + }, + "finished_at": "2026-10-25T14:13:00+04:00", + "wall_secs": 47.6, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..97663dc6da4300d7e158838ac45dc6796b080d10 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761120fb3fd727d5f1c6467dc721a56e7db73bfa1807315860d7606fa1089ee5 +size 327779 diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.json b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.json new file mode 100644 index 0000000000000000000000000000000000000000..b4da76a84d6ade6f29c3b0de89fcdb6f8624e2b3 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.json @@ -0,0 +1,535 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225 + }, + "cost_model": { + "currency": "gpu_s", + "unit": "GPU-seconds", + "per_equiv": 0.0006341154090044388, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6582624885161, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.3165249770323, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.9747874655477, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.6330499540645, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.29131244258, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.949574931095, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.607837419612, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9197.266099908129, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4412.0116364692885, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 5447.452258972225, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5928.330676213297, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5575.824379905499, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.396916034446, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 7354.289774195097, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7941.14717623332, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.669323298678, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 8926.147533635882, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.71692481839, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 10583.520622295184, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 11733.178884783698, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 12882.837147272214, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 8284.204097318152, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.6473977249664 + }, + { + "R": 16000, + "acc": 0.8147501836884644, + "equiv": 1471.577875342962 + }, + { + "R": 24000, + "acc": 0.9170646583394563, + "equiv": 4832.890713124799 + }, + { + "R": 32000, + "acc": 0.9275349008082292, + "equiv": 5413.439101005121 + }, + { + "R": 48000, + "acc": 0.9309331373989713, + "equiv": 5644.2834538675625 + }, + { + "R": 64000, + "acc": 0.9356171932402645, + "equiv": 5862.764293841994 + }, + { + "R": 96000, + "acc": 0.9362601028655401, + "equiv": 6067.768352462251 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9270756796473182, + "equiv": 5420.51610837143, + "R": [ + 43045.42814004174, + 29701.800395449398 + ], + "delta_vs_agree2": [ + -0.002847171197648788, + -0.009185571271124173, + 0.0037667615723732486 + ], + "actions": { + "stop@1": 3414, + "stop@2": 3106, + "stop@4": 147, + "think:think16k@0": 3576, + "stop@5": 136, + "stop@6": 24, + "think:think2k@3": 49, + "think:think2k@4": 30, + "think:think8k@3": 40, + "stop@8": 12, + "think:think2k@7": 11, + "think:think4k@2": 193, + "stop@7": 9, + "think:think4k@4": 20, + "think:think4k@3": 46, + "think:think8k@5": 6, + "think:think8k@7": 1, + "stop@3": 31, + "think:think2k@2": 34, + "think:think4k@6": 2, + "think:think4k@8": 1 + } + }, + { + "seed": 1, + "acc": 0.924044819985305, + "equiv": 5142.965000517061, + "R": [ + 36524.52389399567, + 32147.66121024292 + ], + "delta_vs_agree2": [ + -0.005878030859662013, + -0.014697373254959588, + 0.002801249081557678 + ], + "actions": { + "stop@3": 164, + "stop@7": 25, + "think:think4k@3": 73, + "stop@4": 180, + "stop@2": 2767, + "think:think16k@0": 1560, + "stop@1": 3488, + "think:think4k@2": 53, + "think:think8k@3": 61, + "think:think8k@4": 27, + "stop@5": 57, + "stop@6": 22, + "think:think2k@2": 46, + "think:think4k@7": 5, + "think:think4k@5": 29, + "think:think4k@8": 4, + "think:think8k@5": 14, + "stop@8": 28, + "think:think4k@4": 5, + "think:think4k@6": 2, + "think:think8k@0": 2128, + "think:think8k@2": 94, + "think:think2k@3": 34, + "think:think2k@4": 17, + "think:think2k@6": 2, + "think:think2k@5": 3 + } + }, + { + "seed": 2, + "acc": 0.9289125642909625, + "equiv": 5525.605037887456, + "R": [ + 45866.22401839577, + 37514.35256553762 + ], + "delta_vs_agree2": [ + -0.0010102865540044085, + -0.007027231814842027, + 0.005189199118295371 + ], + "actions": { + "stop@5": 205, + "stop@4": 48, + "stop@2": 3726, + "stop@1": 2778, + "think:think2k@3": 18, + "think:think2k@2": 42, + "think:think2k@4": 8, + "think:think16k@0": 3600, + "stop@3": 30, + "think:think4k@3": 105, + "think:think4k@8": 2, + "think:think16k@7": 1, + "stop@7": 18, + "think:think4k@4": 28, + "think:think8k@3": 57, + "think:think4k@7": 4, + "think:think4k@5": 16, + "stop@6": 23, + "stop@8": 13, + "think:think8k@4": 11, + "think:think16k@8": 2, + "think:think8k@6": 6, + "think:think8k@5": 2, + "think:think2k@7": 1, + "think:think4k@2": 143, + "think:think4k@6": 1 + } + }, + { + "seed": 3, + "acc": 0.9293717854518736, + "equiv": 5602.49590364312, + "R": [ + 56336.30710001925, + 30027.600499281278 + ], + "delta_vs_agree2": [ + -0.0005510653930933138, + -0.006704628949301984, + 0.005648420279206466 + ], + "actions": { + "stop@2": 4225, + "stop@6": 53, + "think:think8k@4": 29, + "think:think8k@3": 49, + "stop@5": 118, + "stop@3": 101, + "think:think4k@2": 146, + "think:think16k@0": 3648, + "think:think4k@3": 49, + "think:think4k@4": 37, + "stop@7": 16, + "stop@4": 125, + "think:think8k@5": 4, + "think:think4k@8": 2, + "think:think2k@4": 5, + "think:think16k@3": 6, + "stop@8": 12, + "think:think2k@7": 6, + "think:think16k@4": 3, + "think:think2k@8": 4, + "think:think4k@6": 2, + "think:think8k@7": 1, + "think:think4k@5": 7, + "think:think2k@2": 165, + "stop@1": 2057, + "think:think2k@3": 17, + "think:think4k@7": 1 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.925789860396767, + "equiv": 5375.070126887772, + "R": [ + 27842.54085224983, + 29701.93139233724 + ], + "delta_acc": [ + -0.004132990448199853, + -0.010883541513592946, + 0.0025257163850110212 + ], + "rel_cost": [ + -0.013287336656367336, + -0.0441290603710765, + 0.019224462994066674 + ] + }, + { + "seed": 1, + "acc": 0.9280859662013226, + "equiv": 5457.448502212623, + "R": [ + 24109.46151639172, + 45343.59400796991 + ], + "delta_acc": [ + -0.0018368846436443791, + -0.00799044819985305, + 0.004132990448199853 + ], + "rel_cost": [ + 0.0018350309034712408, + -0.03025013169504272, + 0.03564791376548525 + ] + }, + { + "seed": 2, + "acc": 0.9271675238795004, + "equiv": 5468.350060644595, + "R": [ + 31705.740983568758, + 37514.288515254826 + ], + "delta_acc": [ + -0.002755326965466569, + -0.008819342395297575, + 0.0034453067597354824 + ], + "rel_cost": [ + 0.0038362523761361774, + -0.02770037913573246, + 0.036521085011271076 + ] + }, + { + "seed": 3, + "acc": 0.9190852314474651, + "equiv": 5130.773651698347, + "R": [ + 48872.09428849942, + 23690.455292808045 + ], + "delta_acc": [ + -0.010837619397501837, + -0.018965833945628216, + -0.002847171197648788 + ], + "rel_cost": [ + -0.05813334238079688, + -0.09205327350614068, + -0.021827501757449425 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.log b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.log new file mode 100644 index 0000000000000000000000000000000000000000..8e7745410abf07ad365f1ad88a35b78b6e433244 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,472 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 5447 +cost model gpu_s: 1 equiv = 0.0006341 GPU-seconds; agree2 costs 3.454 GPU-seconds per problem + agree2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -4298; 0.729 GPU-seconds) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -3148; 1.458 GPU-seconds) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1998; 2.187 GPU-seconds) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4599 ( -849; 2.916 GPU-seconds) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +301; 3.645 GPU-seconds) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +1450; 4.374 GPU-seconds) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8048 ( +2600; 5.103 GPU-seconds) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +3750; 5.832 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -1035; 2.798 GPU-seconds) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 5447 ( +0; 3.454 GPU-seconds) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5928 ( +481; 3.759 GPU-seconds) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5576 ( +128; 3.536 GPU-seconds) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +718; 3.91 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 7354 ( +1907; 4.663 GPU-seconds) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7941 ( +2494; 5.036 GPU-seconds) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7112 ( +1664; 4.51 GPU-seconds) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 8926 ( +3479; 5.66 GPU-seconds) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8342 ( +2894; 5.29 GPU-seconds) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 10584 ( +5136; 6.711 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 11733 ( +6286; 7.44 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 12883 ( +7435; 8.169 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 8284 ( +2837; 5.253 GPU-seconds) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.815 equiv 1472 +R 24000: acc 0.917 equiv 4833 +R 32000: acc 0.928 equiv 5413 +R 48000: acc 0.931 equiv 5644 +R 64000: acc 0.936 equiv 5863 +R 96000: acc 0.936 equiv 6068 +parity: adaptive-SC Bayes at agree2's cost acc 0.927 equiv 5421 (-27) | vs agree2 -0.3 [-0.9, +0.4] | R [43045, 29702] | {'think:think16k@0': 3576, 'stop@1': 3414, 'stop@2': 3106, 'think:think4k@2': 193, 'stop@4': 147, 'stop@5': 136, 'think:think2k@3': 49, 'think:think4k@3': 46} +split 1: adaptive-SC Bayes at agree2's cost acc 0.924 equiv 5143 (-304) | vs agree2 -0.6 [-1.5, +0.3] | R [36525, 32148] | {'stop@1': 3488, 'stop@2': 2767, 'think:think8k@0': 2128, 'think:think16k@0': 1560, 'stop@4': 180, 'stop@3': 164, 'think:think8k@2': 94, 'think:think4k@3': 73} +split 2: adaptive-SC Bayes at agree2's cost acc 0.929 equiv 5526 (+78) | vs agree2 -0.1 [-0.7, +0.5] | R [45866, 37514] | {'stop@2': 3726, 'think:think16k@0': 3600, 'stop@1': 2778, 'stop@5': 205, 'think:think4k@2': 143, 'think:think4k@3': 105, 'think:think8k@3': 57, 'stop@4': 48} +split 3: adaptive-SC Bayes at agree2's cost acc 0.929 equiv 5602 (+155) | vs agree2 -0.1 [-0.7, +0.6] | R [56336, 30028] | {'stop@2': 4225, 'think:think16k@0': 3648, 'stop@1': 2057, 'think:think2k@2': 165, 'think:think4k@2': 146, 'stop@4': 125, 'stop@5': 118, 'stop@3': 101} +parity at agree2's accuracy: acc 0.926 cost 3.408 GPU-seconds vs 3.454 | cost -1.3% [-4.4, +1.9] | acc -0.4 [-1.1, +0.3] +split 1 at agree2's accuracy: acc 0.928 cost 3.461 GPU-seconds vs 3.454 | cost +0.2% [-3.0, +3.6] | acc -0.2 [-0.8, +0.4] +split 2 at agree2's accuracy: acc 0.927 cost 3.468 GPU-seconds vs 3.454 | cost +0.4% [-2.8, +3.7] | acc -0.3 [-0.9, +0.3] +split 3 at agree2's accuracy: acc 0.919 cost 3.254 GPU-seconds vs 3.454 | cost -5.8% [-9.2, -2.2] | acc -1.1 [-1.9, -0.3] diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2838526e59314e39c585345d19f132a3fe5687 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_gpu_s.provenance.json @@ -0,0 +1,108 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "gpu_s", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "think2k=workdir/bayes/think_b2048_tier.json@qwen3-8b:think", + "think4k=workdir/bayes/think_b4096_tier.json@qwen3-8b:think", + "think8k=workdir/bayes/think_b8192_tier.json@qwen3-8b:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scbud_gpu_s.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:think2k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b2048_tier.json", + "bytes": 26686, + "sha256": "5f861b6306025bbea0bac2d80ed1efd426265b93cba2b93cbd9795420d7990ba" + }, + "tier:think4k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b4096_tier.json", + "bytes": 26571, + "sha256": "22589a6ba1e6013b129cbdee8b9c0a50b1c2a1847fea90654a0e57d54420bcc2" + }, + "tier:think8k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b8192_tier.json", + "bytes": 26452, + "sha256": "e1341cda4ac24b250db3f23b0b29db64d54afaed954cdc584437dc415bcc91d4" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.4, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..6928019c4aa131c9107f4cb1c8d780ddbc6ae23a --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f782976d4a0c43b3db44f23c00755deb55a0b7cdcfddc1806d6ccb21ade32e +size 187591 diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.json b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..6bde0fbd191062024fbbba6c331d4a36a62f142e --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.json @@ -0,0 +1,536 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734 + }, + "cost_model": { + "currency": "tokens", + "unit": "base tokens", + "per_equiv": 1.0, + "scenario": null, + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1124.9408523144746, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2249.881704628949, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3374.822556943424, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4499.763409257898, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5624.704261572373, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6749.645113886848, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 7874.585966201323, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 8999.526818515797, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4328.9513225569435, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4659.625211241734, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5133.4060525349005, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 4968.986232549595, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6046.053545187362, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6364.151193975019, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 6941.938739897135, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 6970.480804555474, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7812.266908523145, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8168.442964731815, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8591.141807494489, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9716.082659808964, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10841.023512123438, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6341.26010286554, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7908706833210875, + "equiv": 1125.9641256429097 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1125.8735672299779 + }, + { + "R": 16000, + "acc": 0.8854702424687729, + "equiv": 2947.4475018368844 + }, + { + "R": 24000, + "acc": 0.9226671565025716, + "equiv": 4222.056098457017 + }, + { + "R": 32000, + "acc": 0.9281778104335048, + "equiv": 4389.2692138133725 + }, + { + "R": 48000, + "acc": 0.9342395297575312, + "equiv": 4627.9465099191775 + }, + { + "R": 64000, + "acc": 0.9357549595885378, + "equiv": 4766.697722263041 + }, + { + "R": 96000, + "acc": 0.936627479794269, + "equiv": 5034.967670830272 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9345150624540779, + "equiv": 4634.577204261573, + "R": [ + 51308.29682030946, + 39141.95115169303 + ], + "delta_vs_agree2": [ + 0.004592211609110948, + -0.001423585598824394, + 0.010608008817046289 + ], + "actions": { + "stop@2": 5527, + "think:think4k@2": 504, + "stop@5": 131, + "stop@3": 255, + "think:think16k@0": 3576, + "stop@6": 27, + "think:think4k@4": 38, + "think:think4k@3": 19, + "think:think2k@4": 7, + "think:think2k@3": 13, + "think:think16k@3": 40, + "stop@8": 8, + "think:think16k@7": 1, + "think:think4k@7": 4, + "stop@4": 33, + "stop@7": 26, + "think:think2k@8": 6, + "think:think4k@5": 3, + "think:think2k@6": 1, + "think:think2k@5": 2, + "think:think4k@6": 2, + "think:think8k@5": 6, + "think:think16k@8": 1, + "think:think8k@7": 5, + "think:think8k@6": 1, + "think:think16k@5": 1, + "think:think16k@4": 1, + "stop@1": 585, + "think:think2k@2": 65 + } + }, + { + "seed": 1, + "acc": 0.934974283614989, + "equiv": 4692.648337619397, + "R": [ + 50374.53688391359, + 47085.63614645664 + ], + "delta_vs_agree2": [ + 0.0050514327700220425, + -0.0007829720793534165, + 0.011068378030859657 + ], + "actions": { + "think:think4k@2": 195, + "stop@6": 23, + "think:think4k@3": 64, + "stop@2": 3958, + "think:think16k@0": 3688, + "stop@1": 2121, + "think:think8k@2": 293, + "think:think8k@3": 6, + "stop@4": 15, + "stop@5": 117, + "think:think2k@2": 100, + "stop@3": 154, + "think:think4k@5": 56, + "stop@7": 16, + "think:think4k@7": 1, + "stop@8": 12, + "think:think8k@7": 2, + "think:think8k@6": 3, + "think:think8k@5": 4, + "think:think8k@4": 7, + "think:think4k@6": 1, + "think:think16k@3": 29, + "think:think4k@8": 1, + "think:think16k@8": 2, + "think:think16k@4": 13, + "think:think16k@7": 4, + "think:think16k@5": 3 + } + }, + { + "seed": 2, + "acc": 0.931208670095518, + "equiv": 4652.18438648053, + "R": [ + 55929.90877051365, + 39742.41871531649 + ], + "delta_vs_agree2": [ + 0.0012858192505510655, + -0.004729977957384277, + 0.007255694342395298 + ], + "actions": { + "stop@5": 248, + "stop@8": 81, + "stop@7": 22, + "stop@6": 30, + "stop@4": 21, + "stop@3": 137, + "stop@2": 3947, + "stop@1": 2263, + "think:think2k@2": 66, + "think:think8k@2": 35, + "think:think16k@0": 3600, + "think:think16k@3": 112, + "think:think16k@8": 12, + "think:think16k@7": 7, + "think:think16k@4": 40, + "think:think4k@6": 6, + "think:think16k@5": 14, + "think:think4k@8": 2, + "think:think16k@6": 15, + "think:think2k@3": 2, + "think:think4k@4": 28, + "think:think4k@3": 54, + "think:think4k@2": 143, + "think:think4k@5": 3 + } + }, + { + "seed": 3, + "acc": 0.934147685525349, + "equiv": 4600.5944617928, + "R": [ + 47332.259131041195, + 52114.627727874395 + ], + "delta_vs_agree2": [ + 0.004224834680382072, + -0.0018828067597354886, + 0.010241779941219687 + ], + "actions": { + "stop@2": 4925, + "stop@3": 264, + "think:think8k@4": 14, + "think:think8k@2": 117, + "stop@1": 1164, + "think:think4k@2": 435, + "think:think16k@0": 3648, + "stop@5": 81, + "stop@6": 18, + "think:think4k@3": 40, + "think:think4k@4": 16, + "stop@7": 10, + "think:think8k@6": 3, + "think:think16k@2": 21, + "stop@8": 11, + "stop@4": 29, + "think:think2k@8": 4, + "think:think2k@7": 1, + "think:think8k@5": 1, + "think:think2k@2": 78, + "think:think2k@3": 8 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9256980161645849, + "equiv": 4315.368350477589, + "R": [ + 25232.910463781125, + 22578.147093533287 + ], + "delta_acc": [ + -0.004224834680382072, + -0.011113152094048494, + 0.0023431759735488547 + ], + "rel_cost": [ + -0.07388080481957984, + -0.10175614491720149, + -0.04377170266764283 + ] + }, + { + "seed": 1, + "acc": 0.9301983835415136, + "equiv": 4332.528030859662, + "R": [ + 22808.398423626026, + 26966.932511784937 + ], + "delta_acc": [ + 0.0002755326965466569, + -0.005511801983835415, + 0.0061994856722997795 + ], + "rel_cost": [ + -0.07019817379151498, + -0.0995719765778449, + -0.03958392340853759 + ] + }, + { + "seed": 2, + "acc": 0.9284533431300515, + "equiv": 4357.64077883909, + "R": [ + 30006.314248127055, + 28711.244030847454 + ], + "delta_acc": [ + -0.0014695077149155032, + -0.007439382806759736, + 0.0046840558412931665 + ], + "rel_cost": [ + -0.06480873862431735, + -0.09368118141831566, + -0.03528876265822 + ] + }, + { + "seed": 3, + "acc": 0.9197281410727406, + "equiv": 4099.267312637767, + "R": [ + 42025.52918229907, + 21979.297008870104 + ], + "delta_acc": [ + -0.010194709772226303, + -0.01823337619397502, + -0.002338583761939769 + ], + "rel_cost": [ + -0.12025814806994724, + -0.15185968327555047, + -0.08649069835896024 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.log b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.log new file mode 100644 index 0000000000000000000000000000000000000000..ab1f29b299d0ca7e40d47cb87b24743392b753d8 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,515 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4660 +cost model tokens: 1 equiv = 1 base tokens; agree2 costs 4660 base tokens per problem + agree2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1125 ( -3535; 1125 base tokens) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2250 ( -2410; 2250 base tokens) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3375 ( -1285; 3375 base tokens) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4500 ( -160; 4500 base tokens) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5625 ( +965; 5625 base tokens) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6750 ( +2090; 6750 base tokens) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 7875 ( +3215; 7875 base tokens) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9000 ( +4340; 9000 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4329 ( -331; 4329 base tokens) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4660 ( +0; 4660 base tokens) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5133 ( +474; 5133 base tokens) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 4969 ( +309; 4969 base tokens) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6046 ( +1386; 6046 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6364 ( +1705; 6364 base tokens) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 6942 ( +2282; 6942 base tokens) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 6970 ( +2311; 6970 base tokens) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7812 ( +3153; 7812 base tokens) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8168 ( +3509; 8168 base tokens) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8591 ( +3932; 8591 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9716 ( +5056; 9716 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10841 ( +6181; 1.084e+04 base tokens) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6341 ( +1682; 6341 base tokens) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1126 +R 8000: acc 0.791 equiv 1126 +R 16000: acc 0.885 equiv 2947 +R 24000: acc 0.923 equiv 4222 +R 32000: acc 0.928 equiv 4389 +R 48000: acc 0.934 equiv 4628 +R 64000: acc 0.936 equiv 4767 +R 96000: acc 0.937 equiv 5035 +parity: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4635 (-25) | vs agree2 +0.5 [-0.1, +1.1] | R [51308, 39142] | {'stop@2': 5527, 'think:think16k@0': 3576, 'stop@1': 585, 'think:think4k@2': 504, 'stop@3': 255, 'stop@5': 131, 'think:think2k@2': 65, 'think:think16k@3': 40} +split 1: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4693 (+33) | vs agree2 +0.5 [-0.1, +1.1] | R [50375, 47086] | {'stop@2': 3958, 'think:think16k@0': 3688, 'stop@1': 2121, 'think:think8k@2': 293, 'think:think4k@2': 195, 'stop@3': 154, 'stop@5': 117, 'think:think2k@2': 100} +split 2: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 4652 (-7) | vs agree2 +0.1 [-0.5, +0.7] | R [55930, 39742] | {'stop@2': 3947, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 248, 'think:think4k@2': 143, 'stop@3': 137, 'think:think16k@3': 112, 'stop@8': 81} +split 3: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4601 (-59) | vs agree2 +0.4 [-0.2, +1.0] | R [47332, 52115] | {'stop@2': 4925, 'think:think16k@0': 3648, 'stop@1': 1164, 'think:think4k@2': 435, 'stop@3': 264, 'think:think8k@2': 117, 'stop@5': 81, 'think:think2k@2': 78} +parity at agree2's accuracy: acc 0.926 cost 4315 base tokens vs 4660 | cost -7.4% [-10.2, -4.4] | acc -0.4 [-1.1, +0.2] +split 1 at agree2's accuracy: acc 0.930 cost 4333 base tokens vs 4660 | cost -7.0% [-10.0, -4.0] | acc +0.0 [-0.6, +0.6] +split 2 at agree2's accuracy: acc 0.928 cost 4358 base tokens vs 4660 | cost -6.5% [-9.4, -3.5] | acc -0.1 [-0.7, +0.5] +split 3 at agree2's accuracy: acc 0.920 cost 4099 base tokens vs 4660 | cost -12.0% [-15.2, -8.6] | acc -1.0 [-1.8, -0.2] diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..6dfa3d6226688e24c5ed167573375bd2f5c3ee6a --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_tokens.provenance.json @@ -0,0 +1,108 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "tokens", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json", + "think2k=workdir/bayes/think_b2048_tier.json", + "think4k=workdir/bayes/think_b4096_tier.json", + "think8k=workdir/bayes/think_b8192_tier.json", + "--match-acc", + "--out", + "workdir/bayes/v8/scbud_tokens.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:think2k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b2048_tier.json", + "bytes": 26686, + "sha256": "5f861b6306025bbea0bac2d80ed1efd426265b93cba2b93cbd9795420d7990ba" + }, + "tier:think4k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b4096_tier.json", + "bytes": 26571, + "sha256": "22589a6ba1e6013b129cbdee8b9c0a50b1c2a1847fea90654a0e57d54420bcc2" + }, + "tier:think8k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b8192_tier.json", + "bytes": 26452, + "sha256": "e1341cda4ac24b250db3f23b0b29db64d54afaed954cdc584437dc415bcc91d4" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 44.8, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..a41356e61e6c86c76c088def580dbdfc4ed7be7c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6556272ae0dfe66eeeabed843e4f273f074438dc81a405603b9c3ad7799383c9 +size 181413 diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.json new file mode 100644 index 0000000000000000000000000000000000000000..2cb37afe5ca4b768a736aa6b669fdaa793f9ad95 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.json @@ -0,0 +1,534 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "cheapest", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7908706833210875, + "equiv": 1150.6337711241733 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.5466109478325 + }, + { + "R": 16000, + "acc": 0.8854702424687729, + "equiv": 2966.761572373255 + }, + { + "R": 24000, + "acc": 0.9226671565025716, + "equiv": 4244.129569250551 + }, + { + "R": 32000, + "acc": 0.928269654665687, + "equiv": 4415.09680382072 + }, + { + "R": 48000, + "acc": 0.9342395297575312, + "equiv": 4658.176065393093 + }, + { + "R": 64000, + "acc": 0.9357549595885378, + "equiv": 4800.890544636297 + }, + { + "R": 96000, + "acc": 0.9369030124908155, + "equiv": 5068.57953710507 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9335966201322556, + "equiv": 4611.763202608377, + "R": [ + 53019.483514592255, + 40548.75774047808 + ], + "delta_vs_agree2": [ + 0.0036737692872887582, + -0.0024809423218221894, + 0.009781410727406318 + ], + "actions": { + "stop@1": 1765, + "stop@2": 4382, + "think:think4k@2": 504, + "stop@5": 113, + "stop@3": 175, + "think:think16k@0": 3576, + "stop@6": 27, + "think:think4k@4": 38, + "think:think4k@3": 19, + "think:think2k@4": 4, + "think:think2k@3": 10, + "think:think16k@3": 40, + "stop@8": 9, + "think:think16k@7": 1, + "think:think4k@7": 4, + "stop@4": 96, + "stop@7": 33, + "think:think2k@8": 5, + "think:think4k@5": 7, + "think:think4k@6": 1, + "think:think8k@5": 6, + "think:think8k@7": 5, + "think:think8k@6": 1, + "think:think16k@5": 1, + "think:think16k@4": 1, + "think:think2k@2": 65 + } + }, + { + "seed": 1, + "acc": 0.9355712711241734, + "equiv": 4775.565370132255, + "R": [ + 51981.590411626, + 48418.41812488841 + ], + "delta_vs_agree2": [ + 0.005648420279206466, + -0.00046036921381337247, + 0.01166536554004408 + ], + "actions": { + "think:think4k@2": 195, + "stop@6": 23, + "think:think4k@3": 64, + "stop@2": 4926, + "think:think16k@0": 3688, + "stop@1": 1128, + "think:think8k@2": 293, + "think:think8k@3": 6, + "stop@4": 15, + "stop@5": 117, + "think:think16k@2": 10, + "stop@3": 154, + "think:think4k@5": 56, + "stop@7": 16, + "think:think4k@7": 1, + "stop@8": 12, + "think:think8k@7": 2, + "think:think8k@6": 3, + "think:think8k@5": 4, + "think:think8k@4": 7, + "think:think4k@6": 1, + "think:think16k@3": 29, + "think:think4k@8": 1, + "think:think2k@2": 115, + "think:think16k@8": 2, + "think:think16k@4": 13, + "think:think16k@7": 4, + "think:think16k@5": 3 + } + }, + { + "seed": 2, + "acc": 0.9313005143277002, + "equiv": 4683.506520940485, + "R": [ + 57145.55958989357, + 40486.832067888594 + ], + "delta_vs_agree2": [ + 0.0013776634827332844, + -0.004592211609110948, + 0.007393460690668626 + ], + "actions": { + "stop@5": 248, + "stop@8": 79, + "stop@7": 22, + "stop@6": 30, + "stop@4": 21, + "stop@3": 137, + "stop@2": 3947, + "stop@1": 2263, + "think:think2k@2": 66, + "think:think8k@2": 35, + "think:think16k@0": 3600, + "think:think16k@3": 112, + "think:think16k@8": 12, + "think:think16k@7": 6, + "think:think16k@4": 40, + "think:think4k@6": 6, + "think:think16k@5": 14, + "think:think4k@8": 2, + "think:think16k@6": 18, + "think:think2k@3": 2, + "think:think4k@4": 28, + "think:think4k@3": 54, + "think:think4k@2": 143, + "think:think4k@5": 3 + } + }, + { + "seed": 3, + "acc": 0.9342395297575312, + "equiv": 4642.272157421014, + "R": [ + 50735.22075179102, + 52098.15106542517 + ], + "delta_vs_agree2": [ + 0.004316678912564291, + -0.0017909625275532696, + 0.010379546289493015 + ], + "actions": { + "stop@2": 4925, + "stop@6": 39, + "think:think8k@4": 14, + "think:think8k@2": 117, + "stop@3": 223, + "stop@1": 1164, + "think:think4k@2": 435, + "think:think16k@0": 3648, + "stop@5": 85, + "think:think4k@3": 42, + "think:think4k@4": 16, + "stop@7": 10, + "stop@4": 42, + "think:think8k@6": 4, + "think:think16k@2": 21, + "stop@8": 11, + "think:think2k@8": 4, + "think:think2k@7": 1, + "think:think8k@5": 1, + "think:think2k@2": 78, + "think:think2k@3": 8 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9256980161645849, + "equiv": 4339.151474099926, + "R": [ + 25675.440239418684, + 22570.51369900952 + ], + "delta_acc": [ + -0.004224834680382072, + -0.011113152094048494, + 0.0023431759735488547 + ], + "rel_cost": [ + -0.07884378761049697, + -0.10653790889499846, + -0.04917965667518861 + ] + }, + { + "seed": 1, + "acc": 0.9303820720058781, + "equiv": 4357.511756061719, + "R": [ + 23285.26867045763, + 27496.68910218083 + ], + "delta_acc": [ + 0.0004592211609110948, + -0.005328113519470977, + 0.006337252020573108 + ], + "rel_cost": [ + -0.07494609289039353, + -0.10411614579654693, + -0.0447107431530769 + ] + }, + { + "seed": 2, + "acc": 0.9291880969875091, + "equiv": 4386.020297575313, + "R": [ + 30573.46148315836, + 28701.93440633812 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.006520940484937546, + 0.00541995775165319 + ], + "rel_cost": [ + -0.06889402942172711, + -0.09818762791316155, + -0.039655831309608064 + ] + }, + { + "seed": 3, + "acc": 0.9197281410727406, + "equiv": 4122.9529987141805, + "R": [ + 42013.23164687279, + 22320.76047870937 + ], + "delta_acc": [ + -0.010194709772226303, + -0.01823337619397502, + -0.002338583761939769 + ], + "rel_cost": [ + -0.12474044964210515, + -0.15613965822621984, + -0.09119849602612831 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.log b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.log new file mode 100644 index 0000000000000000000000000000000000000000..8c5a572ed055831f48daa6e1f60986fc2f1d03ee --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,503 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.885 equiv 2967 +R 24000: acc 0.923 equiv 4244 +R 32000: acc 0.928 equiv 4415 +R 48000: acc 0.934 equiv 4658 +R 64000: acc 0.936 equiv 4801 +R 96000: acc 0.937 equiv 5069 +parity: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4612 (-99) | vs agree2 +0.4 [-0.2, +1.0] | R [53019, 40549] | {'stop@2': 4382, 'think:think16k@0': 3576, 'stop@1': 1765, 'think:think4k@2': 504, 'stop@3': 175, 'stop@5': 113, 'stop@4': 96, 'think:think2k@2': 65} +split 1: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4776 (+65) | vs agree2 +0.6 [-0.0, +1.2] | R [51982, 48418] | {'stop@2': 4926, 'think:think16k@0': 3688, 'stop@1': 1128, 'think:think8k@2': 293, 'think:think4k@2': 195, 'stop@3': 154, 'stop@5': 117, 'think:think2k@2': 115} +split 2: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 4684 (-27) | vs agree2 +0.1 [-0.5, +0.7] | R [57146, 40487] | {'stop@2': 3947, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 248, 'think:think4k@2': 143, 'stop@3': 137, 'think:think16k@3': 112, 'stop@8': 79} +split 3: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4642 (-68) | vs agree2 +0.4 [-0.2, +1.0] | R [50735, 52098] | {'stop@2': 4925, 'think:think16k@0': 3648, 'stop@1': 1164, 'think:think4k@2': 435, 'stop@3': 223, 'think:think8k@2': 117, 'stop@5': 85, 'think:think2k@2': 78} +parity at agree2's accuracy: acc 0.926 cost 0.0008678 USD vs 0.0009421 | cost -7.9% [-10.7, -4.9] | acc -0.4 [-1.1, +0.2] +split 1 at agree2's accuracy: acc 0.930 cost 0.0008715 USD vs 0.0009421 | cost -7.5% [-10.4, -4.5] | acc +0.0 [-0.5, +0.6] +split 2 at agree2's accuracy: acc 0.929 cost 0.0008772 USD vs 0.0009421 | cost -6.9% [-9.8, -4.0] | acc -0.1 [-0.7, +0.5] +split 3 at agree2's accuracy: acc 0.920 cost 0.0008246 USD vs 0.0009421 | cost -12.5% [-15.6, -9.1] | acc -1.0 [-1.8, -0.2] diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..5b05d8562483c951a99bf40c95a79a88de1c6491 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_cheapest.provenance.json @@ -0,0 +1,110 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "cheapest", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "think2k=workdir/bayes/think_b2048_tier.json@qwen3-8b:think", + "think4k=workdir/bayes/think_b4096_tier.json@qwen3-8b:think", + "think8k=workdir/bayes/think_b8192_tier.json@qwen3-8b:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scbud_usd_cheapest.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:think2k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b2048_tier.json", + "bytes": 26686, + "sha256": "5f861b6306025bbea0bac2d80ed1efd426265b93cba2b93cbd9795420d7990ba" + }, + "tier:think4k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b4096_tier.json", + "bytes": 26571, + "sha256": "22589a6ba1e6013b129cbdee8b9c0a50b1c2a1847fea90654a0e57d54420bcc2" + }, + "tier:think8k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b8192_tier.json", + "bytes": 26452, + "sha256": "e1341cda4ac24b250db3f23b0b29db64d54afaed954cdc584437dc415bcc91d4" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.0, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..a41356e61e6c86c76c088def580dbdfc4ed7be7c --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6556272ae0dfe66eeeabed843e4f273f074438dc81a405603b9c3ad7799383c9 +size 181413 diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.json new file mode 100644 index 0000000000000000000000000000000000000000..b16f152227c6c9e9d37ef9de533eb79a7488b4db --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.json @@ -0,0 +1,534 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 2.0000000000000002e-07, + "scenario": "dear_32b", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1149.6065393093313, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2299.2130786186626, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3448.819617927994, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4598.426157237325, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5748.032696546657, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6897.639235855988, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8047.245775165319, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9196.85231447465, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4411.8378260470245, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4710.548998897869, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5192.291054371785, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5035.375596987509, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6165.147180382072, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6440.134299228508, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7027.851212343865, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7111.373874908156, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7913.1419911829535, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8341.35433504776, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8645.406318883173, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9795.012858192506, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10944.619397501838, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.193240264512, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7908706833210875, + "equiv": 1150.6337711241733 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1150.5466109478325 + }, + { + "R": 16000, + "acc": 0.8854702424687729, + "equiv": 2966.761572373255 + }, + { + "R": 24000, + "acc": 0.9226671565025716, + "equiv": 4244.129569250551 + }, + { + "R": 32000, + "acc": 0.928269654665687, + "equiv": 4415.09680382072 + }, + { + "R": 48000, + "acc": 0.9342395297575312, + "equiv": 4658.176065393093 + }, + { + "R": 64000, + "acc": 0.9357549595885378, + "equiv": 4800.890544636297 + }, + { + "R": 96000, + "acc": 0.9369030124908155, + "equiv": 5068.57953710507 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9335966201322556, + "equiv": 4611.763202608377, + "R": [ + 53019.483514592255, + 40548.75774047808 + ], + "delta_vs_agree2": [ + 0.0036737692872887582, + -0.0024809423218221894, + 0.009781410727406318 + ], + "actions": { + "stop@1": 1765, + "stop@2": 4382, + "think:think4k@2": 504, + "stop@5": 113, + "stop@3": 175, + "think:think16k@0": 3576, + "stop@6": 27, + "think:think4k@4": 38, + "think:think4k@3": 19, + "think:think2k@4": 4, + "think:think2k@3": 10, + "think:think16k@3": 40, + "stop@8": 9, + "think:think16k@7": 1, + "think:think4k@7": 4, + "stop@4": 96, + "stop@7": 33, + "think:think2k@8": 5, + "think:think4k@5": 7, + "think:think4k@6": 1, + "think:think8k@5": 6, + "think:think8k@7": 5, + "think:think8k@6": 1, + "think:think16k@5": 1, + "think:think16k@4": 1, + "think:think2k@2": 65 + } + }, + { + "seed": 1, + "acc": 0.9355712711241734, + "equiv": 4775.565370132255, + "R": [ + 51981.590411626, + 48418.41812488841 + ], + "delta_vs_agree2": [ + 0.005648420279206466, + -0.00046036921381337247, + 0.01166536554004408 + ], + "actions": { + "think:think4k@2": 195, + "stop@6": 23, + "think:think4k@3": 64, + "stop@2": 4926, + "think:think16k@0": 3688, + "stop@1": 1128, + "think:think8k@2": 293, + "think:think8k@3": 6, + "stop@4": 15, + "stop@5": 117, + "think:think16k@2": 10, + "stop@3": 154, + "think:think4k@5": 56, + "stop@7": 16, + "think:think4k@7": 1, + "stop@8": 12, + "think:think8k@7": 2, + "think:think8k@6": 3, + "think:think8k@5": 4, + "think:think8k@4": 7, + "think:think4k@6": 1, + "think:think16k@3": 29, + "think:think4k@8": 1, + "think:think2k@2": 115, + "think:think16k@8": 2, + "think:think16k@4": 13, + "think:think16k@7": 4, + "think:think16k@5": 3 + } + }, + { + "seed": 2, + "acc": 0.9313005143277002, + "equiv": 4683.506520940485, + "R": [ + 57145.55958989357, + 40486.832067888594 + ], + "delta_vs_agree2": [ + 0.0013776634827332844, + -0.004592211609110948, + 0.007393460690668626 + ], + "actions": { + "stop@5": 248, + "stop@8": 79, + "stop@7": 22, + "stop@6": 30, + "stop@4": 21, + "stop@3": 137, + "stop@2": 3947, + "stop@1": 2263, + "think:think2k@2": 66, + "think:think8k@2": 35, + "think:think16k@0": 3600, + "think:think16k@3": 112, + "think:think16k@8": 12, + "think:think16k@7": 6, + "think:think16k@4": 40, + "think:think4k@6": 6, + "think:think16k@5": 14, + "think:think4k@8": 2, + "think:think16k@6": 18, + "think:think2k@3": 2, + "think:think4k@4": 28, + "think:think4k@3": 54, + "think:think4k@2": 143, + "think:think4k@5": 3 + } + }, + { + "seed": 3, + "acc": 0.9342395297575312, + "equiv": 4642.272157421014, + "R": [ + 50735.22075179102, + 52098.15106542517 + ], + "delta_vs_agree2": [ + 0.004316678912564291, + -0.0017909625275532696, + 0.010379546289493015 + ], + "actions": { + "stop@2": 4925, + "stop@6": 39, + "think:think8k@4": 14, + "think:think8k@2": 117, + "stop@3": 223, + "stop@1": 1164, + "think:think4k@2": 435, + "think:think16k@0": 3648, + "stop@5": 85, + "think:think4k@3": 42, + "think:think4k@4": 16, + "stop@7": 10, + "stop@4": 42, + "think:think8k@6": 4, + "think:think16k@2": 21, + "stop@8": 11, + "think:think2k@8": 4, + "think:think2k@7": 1, + "think:think8k@5": 1, + "think:think2k@2": 78, + "think:think2k@3": 8 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9256980161645849, + "equiv": 4339.151474099926, + "R": [ + 25675.440239418684, + 22570.51369900952 + ], + "delta_acc": [ + -0.004224834680382072, + -0.011113152094048494, + 0.0023431759735488547 + ], + "rel_cost": [ + -0.07884378761049697, + -0.10653790889499846, + -0.04917965667518861 + ] + }, + { + "seed": 1, + "acc": 0.9303820720058781, + "equiv": 4357.511756061719, + "R": [ + 23285.26867045763, + 27496.68910218083 + ], + "delta_acc": [ + 0.0004592211609110948, + -0.005328113519470977, + 0.006337252020573108 + ], + "rel_cost": [ + -0.07494609289039353, + -0.10411614579654693, + -0.0447107431530769 + ] + }, + { + "seed": 2, + "acc": 0.9291880969875091, + "equiv": 4386.020297575313, + "R": [ + 30573.46148315836, + 28701.93440633812 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.006520940484937546, + 0.00541995775165319 + ], + "rel_cost": [ + -0.06889402942172711, + -0.09818762791316155, + -0.039655831309608064 + ] + }, + { + "seed": 3, + "acc": 0.9197281410727406, + "equiv": 4122.9529987141805, + "R": [ + 42013.23164687279, + 22320.76047870937 + ], + "delta_acc": [ + -0.010194709772226303, + -0.01823337619397502, + -0.002338583761939769 + ], + "rel_cost": [ + -0.12474044964210515, + -0.15613965822621984, + -0.09119849602612831 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.log b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.log new file mode 100644 index 0000000000000000000000000000000000000000..d1612d38c90c0d15af841d5765fac5c93062ed49 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,504 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4711 +cost model usd: 1 equiv = 2e-07 USD; agree2 costs 0.0009421 USD per problem + agree2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3561; 0.0002299 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2299 ( -2411; 0.0004598 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3449 ( -1262; 0.0006898 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4598 ( -112; 0.0009197 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5748 ( +1037; 0.00115 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6898 ( +2187; 0.00138 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8047 ( +3337; 0.001609 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9197 ( +4486; 0.001839 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4412 ( -299; 0.0008824 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4711 ( +0; 0.0009421 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5192 ( +482; 0.001038 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5035 ( +325; 0.001007 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6165 ( +1455; 0.001233 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6440 ( +1730; 0.001288 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7028 ( +2317; 0.001406 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7111 ( +2401; 0.001422 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7913 ( +3203; 0.001583 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8341 ( +3631; 0.001668 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8645 ( +3935; 0.001729 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9795 ( +5084; 0.001959 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10945 ( +6234; 0.002189 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6346 ( +1636; 0.001269 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.885 equiv 2967 +R 24000: acc 0.923 equiv 4244 +R 32000: acc 0.928 equiv 4415 +R 48000: acc 0.934 equiv 4658 +R 64000: acc 0.936 equiv 4801 +R 96000: acc 0.937 equiv 5069 +parity: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4612 (-99) | vs agree2 +0.4 [-0.2, +1.0] | R [53019, 40549] | {'stop@2': 4382, 'think:think16k@0': 3576, 'stop@1': 1765, 'think:think4k@2': 504, 'stop@3': 175, 'stop@5': 113, 'stop@4': 96, 'think:think2k@2': 65} +split 1: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4776 (+65) | vs agree2 +0.6 [-0.0, +1.2] | R [51982, 48418] | {'stop@2': 4926, 'think:think16k@0': 3688, 'stop@1': 1128, 'think:think8k@2': 293, 'think:think4k@2': 195, 'stop@3': 154, 'stop@5': 117, 'think:think2k@2': 115} +split 2: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 4684 (-27) | vs agree2 +0.1 [-0.5, +0.7] | R [57146, 40487] | {'stop@2': 3947, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 248, 'think:think4k@2': 143, 'stop@3': 137, 'think:think16k@3': 112, 'stop@8': 79} +split 3: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4642 (-68) | vs agree2 +0.4 [-0.2, +1.0] | R [50735, 52098] | {'stop@2': 4925, 'think:think16k@0': 3648, 'stop@1': 1164, 'think:think4k@2': 435, 'stop@3': 223, 'think:think8k@2': 117, 'stop@5': 85, 'think:think2k@2': 78} +parity at agree2's accuracy: acc 0.926 cost 0.0008678 USD vs 0.0009421 | cost -7.9% [-10.7, -4.9] | acc -0.4 [-1.1, +0.2] +split 1 at agree2's accuracy: acc 0.930 cost 0.0008715 USD vs 0.0009421 | cost -7.5% [-10.4, -4.5] | acc +0.0 [-0.5, +0.6] +split 2 at agree2's accuracy: acc 0.929 cost 0.0008772 USD vs 0.0009421 | cost -6.9% [-9.8, -4.0] | acc -0.1 [-0.7, +0.5] +split 3 at agree2's accuracy: acc 0.920 cost 0.0008246 USD vs 0.0009421 | cost -12.5% [-15.6, -9.1] | acc -1.0 [-1.8, -0.2] diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..48375d2cdaf5a633be74132cf5eb32133ff6f4bf --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_dear_32b.provenance.json @@ -0,0 +1,110 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "dear_32b", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "think2k=workdir/bayes/think_b2048_tier.json@qwen3-8b:think", + "think4k=workdir/bayes/think_b4096_tier.json@qwen3-8b:think", + "think8k=workdir/bayes/think_b8192_tier.json@qwen3-8b:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scbud_usd_dear_32b.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:think2k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b2048_tier.json", + "bytes": 26686, + "sha256": "5f861b6306025bbea0bac2d80ed1efd426265b93cba2b93cbd9795420d7990ba" + }, + "tier:think4k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b4096_tier.json", + "bytes": 26571, + "sha256": "22589a6ba1e6013b129cbdee8b9c0a50b1c2a1847fea90654a0e57d54420bcc2" + }, + "tier:think8k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b8192_tier.json", + "bytes": 26452, + "sha256": "e1341cda4ac24b250db3f23b0b29db64d54afaed954cdc584437dc415bcc91d4" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.6, + "n_problems": 1361 +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.arrays.npz b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.arrays.npz new file mode 100644 index 0000000000000000000000000000000000000000..eca843f261da0f2eb0d65cec2d8f6517d417c7d5 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.arrays.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cca02f4394752a229388c47da5cd26218796bed90816fffb78a8da9c75411f18 +size 226820 diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.json new file mode 100644 index 0000000000000000000000000000000000000000..a8221029bd6fbea093dd2d84aad312d305ceb69d --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.json @@ -0,0 +1,537 @@ +{ + "ref": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785 + }, + "cost_model": { + "currency": "usd", + "unit": "USD", + "per_equiv": 4.5500000000000004e-07, + "scenario": "openrouter_only", + "base": "qwen3-8b:nothink" + }, + "baselines": { + "agree2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "SC@1": { + "acc": 0.7907788390889052, + "equiv": 1150.3112732234702, + "delta_vs_agree2": [ + -0.13914401175606173, + -0.15136044268919913, + -0.1273879500367377 + ] + }, + "SC@2": { + "acc": 0.8118111682586333, + "equiv": 2300.6225464469403, + "delta_vs_agree2": [ + -0.11811168258633357, + -0.13019034717119765, + -0.10626262858192506 + ] + }, + "SC@3": { + "acc": 0.8444158706833211, + "equiv": 3450.9338196704107, + "delta_vs_agree2": [ + -0.08550698016164585, + -0.09758908890521675, + -0.07425606171932403 + ] + }, + "SC@4": { + "acc": 0.8553453343130052, + "equiv": 4601.245092893881, + "delta_vs_agree2": [ + -0.07457751653196179, + -0.08610396767083027, + -0.06350799044819987 + ] + }, + "SC@5": { + "acc": 0.8630602498163116, + "equiv": 5751.556366117351, + "delta_vs_agree2": [ + -0.0668626010286554, + -0.07866458486407053, + -0.055564612417340196 + ] + }, + "SC@6": { + "acc": 0.8693056576047025, + "equiv": 6901.867639340821, + "delta_vs_agree2": [ + -0.06061719324026451, + -0.07214594048493755, + -0.04968428545187364 + ] + }, + "SC@7": { + "acc": 0.8717854518736223, + "equiv": 8052.178912564291, + "delta_vs_agree2": [ + -0.0581373989713446, + -0.0704009000734754, + -0.04670279206465834 + ] + }, + "SC@8": { + "acc": 0.8761021307861866, + "equiv": 9202.490185787761, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.8)": { + "acc": 0.8714180749448934, + "equiv": 4414.206011861026, + "delta_vs_agree2": [ + -0.058504775900073476, + -0.07026313372520206, + -0.04679463629684056 + ] + }, + "AC(0.8)+think@2": { + "acc": 0.929922850844967, + "equiv": 4712.1859543927785, + "delta_vs_agree2": [ + 0.0, + 0.0, + 0.0 + ] + }, + "AC(0.8)+think@3": { + "acc": 0.9297391623806025, + "equiv": 5194.155375511704, + "delta_vs_agree2": [ + -0.0001836884643644379, + -0.0005510653930933138, + 0.0 + ] + }, + "AC(0.8)+think@4": { + "acc": 0.9239529757531227, + "equiv": 5037.402507347539, + "delta_vs_agree2": [ + -0.005969875091844232, + -0.008357825128581926, + -0.003857457751653196 + ] + }, + "AC(0.9)": { + "acc": 0.8757347538574578, + "equiv": 6168.549855673349, + "delta_vs_agree2": [ + -0.054188096987509184, + -0.06658821638501101, + -0.04242973916238061 + ] + }, + "AC(0.9)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.9)+think@3": { + "acc": 0.9418626010286554, + "equiv": 6442.532221580771, + "delta_vs_agree2": [ + 0.011939750183688464, + 0.009504729977957385, + 0.014511388684790596 + ] + }, + "AC(0.9)+think@4": { + "acc": 0.9416789125642909, + "equiv": 7030.53273328435, + "delta_vs_agree2": [ + 0.011756061719324026, + 0.009368111682586333, + 0.014281778104335048 + ] + }, + "AC(0.95)": { + "acc": 0.8760102865540044, + "equiv": 7115.399391203946, + "delta_vs_agree2": [ + -0.05391256429096253, + -0.06658706833210874, + -0.04192459588537841 + ] + }, + "AC(0.95)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.95)+think@4": { + "acc": 0.9447556943423953, + "equiv": 7916.276974650992, + "delta_vs_agree2": [ + 0.01483284349742836, + 0.011572373254959589, + 0.018369994489346064 + ] + }, + "AC(0.99)": { + "acc": 0.8761021307861866, + "equiv": 8346.294659913927, + "delta_vs_agree2": [ + -0.05382072005878031, + -0.0664963721528288, + -0.041696133357825134 + ] + }, + "AC(0.99)+think@2": { + "acc": 0.9478324761204996, + "equiv": 8647.52052062559, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@3": { + "acc": 0.9478324761204996, + "equiv": 9797.83179384906, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "AC(0.99)+think@4": { + "acc": 0.9478324761204996, + "equiv": 10948.143067072531, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + }, + "always think": { + "acc": 0.9478324761204996, + "equiv": 6346.89797417865, + "delta_vs_agree2": [ + 0.017909625275532696, + 0.0124896675238795, + 0.023925422483468038 + ] + } + }, + "frontier": [ + { + "R": 4000, + "acc": 0.7908706833210875, + "equiv": 1151.3388081242783 + }, + { + "R": 8000, + "acc": 0.7907788390889052, + "equiv": 1151.2516164584863 + }, + { + "R": 16000, + "acc": 0.8854702424687729, + "equiv": 2967.624787446205 + }, + { + "R": 24000, + "acc": 0.9226671565025716, + "equiv": 4245.050228298521 + }, + { + "R": 32000, + "acc": 0.928269654665687, + "equiv": 4416.118245512754 + }, + { + "R": 48000, + "acc": 0.9342395297575312, + "equiv": 4659.3608061299465 + }, + { + "R": 64000, + "acc": 0.9357549595885378, + "equiv": 4802.180581505197 + }, + { + "R": 96000, + "acc": 0.9369030124908155, + "equiv": 5070.016056995906 + } + ], + "matched": [ + { + "seed": 0, + "acc": 0.9346069066862601, + "equiv": 4666.833835415136, + "R": [ + 53068.75473883116, + 40577.08288223768 + ], + "delta_vs_agree2": [ + 0.0046840558412931665, + -0.0012869673034533432, + 0.010655078986039672 + ], + "actions": { + "stop@2": 5527, + "think:think4k@2": 504, + "stop@5": 131, + "stop@3": 255, + "think:think16k@0": 3576, + "stop@6": 27, + "think:think4k@4": 38, + "think:think4k@3": 19, + "think:think2k@4": 7, + "think:think2k@3": 13, + "think:think16k@3": 40, + "stop@8": 8, + "think:think16k@7": 1, + "think:think4k@7": 3, + "stop@4": 33, + "stop@7": 26, + "think:think2k@8": 5, + "think:think4k@5": 7, + "think:think2k@6": 1, + "think:think2k@5": 2, + "think:think4k@6": 1, + "think:think8k@5": 6, + "think:think8k@7": 5, + "think:think8k@6": 1, + "think:think16k@5": 1, + "think:think16k@4": 1, + "stop@1": 585, + "think:think2k@2": 65 + } + }, + { + "seed": 1, + "acc": 0.9363060249816312, + "equiv": 4820.5258593996, + "R": [ + 52028.18334629778, + 48457.986362529016 + ], + "delta_vs_agree2": [ + 0.006383174136664217, + 0.0002755326965466569, + 0.012124586700955174 + ], + "actions": { + "think:think4k@2": 195, + "stop@6": 23, + "think:think4k@3": 70, + "stop@2": 5462, + "think:think16k@0": 3688, + "think:think8k@2": 293, + "think:think8k@3": 6, + "stop@4": 17, + "stop@5": 118, + "think:think16k@2": 10, + "stop@3": 174, + "think:think8k@5": 5, + "think:think4k@5": 56, + "stop@7": 16, + "think:think4k@7": 1, + "stop@8": 12, + "think:think8k@7": 2, + "think:think8k@6": 3, + "think:think4k@4": 1, + "think:think8k@4": 7, + "think:think4k@6": 1, + "stop@1": 561, + "think:think16k@3": 29, + "think:think4k@8": 1, + "think:think2k@2": 115, + "think:think16k@8": 2, + "think:think16k@4": 13, + "think:think16k@7": 4, + "think:think16k@5": 3 + } + }, + { + "seed": 2, + "acc": 0.9313005143277002, + "equiv": 4684.674477799937, + "R": [ + 57170.41139119937, + 40509.509368195715 + ], + "delta_vs_agree2": [ + 0.0013776634827332844, + -0.004592211609110948, + 0.007393460690668626 + ], + "actions": { + "stop@5": 248, + "stop@8": 79, + "stop@7": 22, + "stop@6": 30, + "stop@4": 21, + "stop@3": 137, + "stop@2": 3947, + "stop@1": 2263, + "think:think2k@2": 66, + "think:think8k@2": 35, + "think:think16k@0": 3600, + "think:think16k@3": 112, + "think:think16k@8": 12, + "think:think16k@7": 6, + "think:think16k@4": 40, + "think:think4k@6": 6, + "think:think16k@5": 14, + "think:think4k@8": 2, + "think:think16k@6": 18, + "think:think2k@3": 2, + "think:think4k@4": 28, + "think:think4k@3": 54, + "think:think4k@2": 143, + "think:think4k@5": 3 + } + }, + { + "seed": 3, + "acc": 0.9342395297575312, + "equiv": 4631.151853941429, + "R": [ + 50773.67186733312, + 52095.74865409548 + ], + "delta_vs_agree2": [ + 0.004316678912564291, + -0.0017909625275532696, + 0.010379546289493015 + ], + "actions": { + "stop@2": 4925, + "stop@3": 264, + "think:think8k@4": 14, + "think:think8k@2": 117, + "stop@1": 1164, + "think:think4k@2": 435, + "think:think16k@0": 3648, + "stop@5": 79, + "stop@6": 18, + "think:think4k@3": 42, + "think:think4k@4": 16, + "stop@7": 10, + "think:think8k@6": 3, + "think:think16k@2": 21, + "stop@8": 11, + "stop@4": 29, + "think:think2k@8": 4, + "think:think2k@7": 1, + "think:think8k@5": 1, + "think:think2k@2": 78, + "think:think2k@3": 8 + } + } + ], + "matched_acc": [ + { + "seed": 0, + "acc": 0.9256980161645849, + "equiv": 4340.126905111788, + "R": [ + 25689.81274830019, + 22569.397921911408 + ], + "delta_acc": [ + -0.004224834680382072, + -0.011113152094048494, + 0.0023431759735488547 + ], + "rel_cost": [ + -0.07895678415113293, + -0.10665022491605859, + -0.04930666177918934 + ] + }, + { + "seed": 1, + "acc": 0.9303820720058781, + "equiv": 4358.465965151675, + "R": [ + 23299.17858964539, + 27512.625147982053 + ], + "delta_acc": [ + 0.0004592211609110948, + -0.005328113519470977, + 0.006337252020573108 + ], + "rel_cost": [ + -0.07506494706800791, + -0.10422769336738905, + -0.0448398381647648 + ] + }, + { + "seed": 2, + "acc": 0.9291880969875091, + "equiv": 4386.99292012176, + "R": [ + 30588.761098112427, + 28700.57227548696 + ], + "delta_acc": [ + -0.0007347538574577516, + -0.006520940484937546, + 0.00541995775165319 + ], + "rel_cost": [ + -0.06901107838663897, + -0.09830143446651103, + -0.03977880116107103 + ] + }, + { + "seed": 3, + "acc": 0.9197281410727406, + "equiv": 4123.942270651832, + "R": [ + 42011.48701785478, + 22331.70946441952 + ], + "delta_acc": [ + -0.010194709772226303, + -0.01823337619397502, + -0.002338583761939769 + ], + "rel_cost": [ + -0.12483456498412933, + -0.15622605814041127, + -0.09130090230637808 + ] + } + ] +} \ No newline at end of file diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.log b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.log new file mode 100644 index 0000000000000000000000000000000000000000..d1b4913c5b89cf8138477de91e0b4f56cb20fe26 --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.log @@ -0,0 +1,45 @@ +2026-10-25 14:12:15,483 online_rl_train INFO: ThinkBooster math evaluator loaded +reference agree2 (rotation pairs, -> think16k): acc 0.930 equiv 4712 +cost model usd: 1 equiv = 4.55e-07 USD; agree2 costs 0.002144 USD per problem + agree2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + SC@1 acc 0.791 equiv 1150 ( -3562; 0.0005234 USD) | vs agree2 -13.9 [-15.1, -12.7] + SC@2 acc 0.812 equiv 2301 ( -2412; 0.001047 USD) | vs agree2 -11.8 [-13.0, -10.6] + SC@3 acc 0.844 equiv 3451 ( -1261; 0.00157 USD) | vs agree2 -8.6 [-9.8, -7.4] + SC@4 acc 0.855 equiv 4601 ( -111; 0.002094 USD) | vs agree2 -7.5 [-8.6, -6.4] + SC@5 acc 0.863 equiv 5752 ( +1039; 0.002617 USD) | vs agree2 -6.7 [-7.9, -5.6] + SC@6 acc 0.869 equiv 6902 ( +2190; 0.00314 USD) | vs agree2 -6.1 [-7.2, -5.0] + SC@7 acc 0.872 equiv 8052 ( +3340; 0.003664 USD) | vs agree2 -5.8 [-7.0, -4.7] + SC@8 acc 0.876 equiv 9202 ( +4490; 0.004187 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.8) acc 0.871 equiv 4414 ( -298; 0.002008 USD) | vs agree2 -5.9 [-7.0, -4.7] + AC(0.8)+think@2 acc 0.930 equiv 4712 ( +0; 0.002144 USD) | vs agree2 +0.0 [+0.0, +0.0] + AC(0.8)+think@3 acc 0.930 equiv 5194 ( +482; 0.002363 USD) | vs agree2 -0.0 [-0.1, +0.0] + AC(0.8)+think@4 acc 0.924 equiv 5037 ( +325; 0.002292 USD) | vs agree2 -0.6 [-0.8, -0.4] + AC(0.9) acc 0.876 equiv 6169 ( +1456; 0.002807 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.9)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.9)+think@3 acc 0.942 equiv 6443 ( +1730; 0.002931 USD) | vs agree2 +1.2 [+1.0, +1.5] + AC(0.9)+think@4 acc 0.942 equiv 7031 ( +2318; 0.003199 USD) | vs agree2 +1.2 [+0.9, +1.4] + AC(0.95) acc 0.876 equiv 7115 ( +2403; 0.003238 USD) | vs agree2 -5.4 [-6.7, -4.2] + AC(0.95)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.95)+think@4 acc 0.945 equiv 7916 ( +3204; 0.003602 USD) | vs agree2 +1.5 [+1.2, +1.8] + AC(0.99) acc 0.876 equiv 8346 ( +3634; 0.003798 USD) | vs agree2 -5.4 [-6.6, -4.2] + AC(0.99)+think@2 acc 0.948 equiv 8648 ( +3935; 0.003935 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@3 acc 0.948 equiv 9798 ( +5086; 0.004458 USD) | vs agree2 +1.8 [+1.2, +2.4] + AC(0.99)+think@4 acc 0.948 equiv 10948 ( +6236; 0.004981 USD) | vs agree2 +1.8 [+1.2, +2.4] + always think acc 0.948 equiv 6347 ( +1635; 0.002888 USD) | vs agree2 +1.8 [+1.2, +2.4] +R 4000: acc 0.791 equiv 1151 +R 8000: acc 0.791 equiv 1151 +R 16000: acc 0.885 equiv 2968 +R 24000: acc 0.923 equiv 4245 +R 32000: acc 0.928 equiv 4416 +R 48000: acc 0.934 equiv 4659 +R 64000: acc 0.936 equiv 4802 +R 96000: acc 0.937 equiv 5070 +parity: adaptive-SC Bayes at agree2's cost acc 0.935 equiv 4667 (-45) | vs agree2 +0.5 [-0.1, +1.1] | R [53069, 40577] | {'stop@2': 5527, 'think:think16k@0': 3576, 'stop@1': 585, 'think:think4k@2': 504, 'stop@3': 255, 'stop@5': 131, 'think:think2k@2': 65, 'think:think16k@3': 40} +split 1: adaptive-SC Bayes at agree2's cost acc 0.936 equiv 4821 (+108) | vs agree2 +0.6 [+0.0, +1.2] | R [52028, 48458] | {'stop@2': 5462, 'think:think16k@0': 3688, 'stop@1': 561, 'think:think8k@2': 293, 'think:think4k@2': 195, 'stop@3': 174, 'stop@5': 118, 'think:think2k@2': 115} +split 2: adaptive-SC Bayes at agree2's cost acc 0.931 equiv 4685 (-28) | vs agree2 +0.1 [-0.5, +0.7] | R [57170, 40510] | {'stop@2': 3947, 'think:think16k@0': 3600, 'stop@1': 2263, 'stop@5': 248, 'think:think4k@2': 143, 'stop@3': 137, 'think:think16k@3': 112, 'stop@8': 79} +split 3: adaptive-SC Bayes at agree2's cost acc 0.934 equiv 4631 (-81) | vs agree2 +0.4 [-0.2, +1.0] | R [50774, 52096] | {'stop@2': 4925, 'think:think16k@0': 3648, 'stop@1': 1164, 'think:think4k@2': 435, 'stop@3': 264, 'think:think8k@2': 117, 'stop@5': 79, 'think:think2k@2': 78} +parity at agree2's accuracy: acc 0.926 cost 0.001975 USD vs 0.002144 | cost -7.9% [-10.7, -4.9] | acc -0.4 [-1.1, +0.2] +split 1 at agree2's accuracy: acc 0.930 cost 0.001983 USD vs 0.002144 | cost -7.5% [-10.4, -4.5] | acc +0.0 [-0.5, +0.6] +split 2 at agree2's accuracy: acc 0.929 cost 0.001996 USD vs 0.002144 | cost -6.9% [-9.8, -4.0] | acc -0.1 [-0.7, +0.5] +split 3 at agree2's accuracy: acc 0.920 cost 0.001876 USD vs 0.002144 | cost -12.5% [-15.6, -9.1] | acc -1.0 [-1.8, -0.2] diff --git a/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.provenance.json b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f50be962ac8919a7f4cf82c32f43f8760324dc7b --- /dev/null +++ b/analysis/bayes_sc_2026-10-25/v8/scbud_usd_openrouter_only.provenance.json @@ -0,0 +1,110 @@ +{ + "started_at": "2026-10-25T14:12:13+04:00", + "host": "ws006600", + "user": "vlad.smirnov", + "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-combos", + "argv": [ + "/mnt/data/users/vlad.smirnov/rl-uhead-combos/rl_controller/scripts/bayes_sc.py", + "--pool-dir", + "workdir/selector/pool_mth_k8_chat", + "--uhead", + "workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "--task-prior", + "workdir/bayes/qprior/mth_qprior.json", + "--cost-model", + "usd", + "--price-scenario", + "openrouter_only", + "--tier", + "think16k=workdir/layer_probe/think16k_tier.json@qwen3-8b:think", + "think2k=workdir/bayes/think_b2048_tier.json@qwen3-8b:think", + "think4k=workdir/bayes/think_b4096_tier.json@qwen3-8b:think", + "think8k=workdir/bayes/think_b8192_tier.json@qwen3-8b:think", + "--match-acc", + "--out", + "workdir/bayes/v8/scbud_usd_openrouter_only.json" + ], + "git": { + "commit": "f935c73993aae0c1137fdec2866aef2005007331", + "branch": "HEAD", + "subject": "Per-problem arrays in the Bayesian controllers; paired bootstrap against the hull mixture in currency_table", + "dirty": false, + "dirty_files": [] + }, + "gpus": { + "visible": "", + "devices": [ + { + "index": "0", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-a15d9536-8962-7d07-d199-3fb785e80c83" + }, + { + "index": "1", + "name": "NVIDIA RTX 5880 Ada Generation", + "memory": "49140 MiB", + "driver": "580.178.04", + "uuid": "GPU-168c29de-060d-b509-1ebd-6be5535c9d6a" + } + ] + }, + "packages": { + "python": "3.11.15", + "torch": "2.9.1", + "vllm": "0.15.1", + "transformers": "4.57.6", + "speculators": "0.4.0.dev49", + "peft": null, + "accelerate": "1.13.0", + "numpy": "2.2.6", + "huggingface_hub": "0.36.2", + "openai": "2.32.0", + "wandb": "0.25.1" + }, + "env": { + "CUDA_VISIBLE_DEVICES": "" + }, + "models": [], + "files": { + "pool": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/pool_chains.jsonl", + "bytes": 65136887, + "sha256": "00f58ec1868cbe5ed10312cfa5649745c18e4a0bd121ee93bd40f0303ece4fda" + }, + "uhead": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/selector/pool_mth_k8_chat/uhead_selector.json", + "bytes": 9369632, + "sha256": "9417d517828e3fc083b56dcb62dada5aeaec69d60684adafe2ce76807fd39e21" + }, + "task_prior": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/qprior/mth_qprior.json", + "bytes": 28987, + "sha256": "897f405c643430dbe97023d9f8985e108330b82ed412a25ab2a4b84c992fbbd6" + }, + "tier:think16k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/layer_probe/think16k_tier.json", + "bytes": 26666, + "sha256": "fc3e1788b20be07c0b78bc3fad95f06ec7eb60258556a8f3c4373d16f23c557f" + }, + "tier:think2k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b2048_tier.json", + "bytes": 26686, + "sha256": "5f861b6306025bbea0bac2d80ed1efd426265b93cba2b93cbd9795420d7990ba" + }, + "tier:think4k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b4096_tier.json", + "bytes": 26571, + "sha256": "22589a6ba1e6013b129cbdee8b9c0a50b1c2a1847fea90654a0e57d54420bcc2" + }, + "tier:think8k": { + "path": "/mnt/data/users/vlad.smirnov/rl-uhead-ood/workdir/bayes/think_b8192_tier.json", + "bytes": 26452, + "sha256": "e1341cda4ac24b250db3f23b0b29db64d54afaed954cdc584437dc415bcc91d4" + } + }, + "finished_at": "2026-10-25T14:12:58+04:00", + "wall_secs": 45.1, + "n_problems": 1361 +} \ No newline at end of file