diff --git a/results/ablations/ablation_data.txt b/results/ablations/ablation_data.txt new file mode 100644 index 0000000000000000000000000000000000000000..eabf66c90993e533d4b88425830f3e46fc2e2bf7 --- /dev/null +++ b/results/ablations/ablation_data.txt @@ -0,0 +1,18 @@ +=== DATA ABLATION (pendulum, N=40, K=8, shared cached oracle, eval noise seed=0, 3 training seeds=[42, 43, 44]) === + +- 25% data (650/2600 examples) + params=3,823,883 time_to_best=261.8s + median regret vs oracle: +0.29% +/- 0.27pp over seeds [s42=+0.61%, s43=+0.16%, s44=+0.12%] + coverage (mean over seeds): certified>=1/plant 0.992 stable>=1/plant 1.000 per-sample stable 1.000 +- 50% data (1300/2600 examples) + params=3,823,883 time_to_best=355.6s + median regret vs oracle: +1.00% +/- 0.23pp over seeds [s42=+1.22%, s43=+0.76%, s44=+1.02%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 +- 100% data (= size_ref, same 3 seeds, reused) + params=3,823,883 time_to_best=421.4s + median regret vs oracle: +0.92% +/- 0.22pp over seeds [s42=+1.06%, s43=+0.67%, s44=+1.03%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 +- hidden=256 (DEPLOYED checkpoint -- pre-existing train_pendulum.py run, not part of the 3-seed sweep) + params=3,823,883 time_to_best=232.0s + median regret vs oracle: +1.15% +/- 0.00pp over seeds [ssingle=+1.15%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 diff --git a/results/ablations/ablation_generative_pendulum.txt b/results/ablations/ablation_generative_pendulum.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a0fdbd9f4eda12ac06ff1f9047878fdf1f6390b --- /dev/null +++ b/results/ablations/ablation_generative_pendulum.txt @@ -0,0 +1,14 @@ +=== A3 GENERATIVE BASELINE — PENDULUM (N=40, shared cached oracle, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=3,823,883 epochs=800 time_to_best=232.0s; cvae params=277,019 epochs=800 time_to_best=278.9s + +-- K=1 -- + diffusion : stable>=1 1.000+-0.000 cert>=1 0.992+-0.014 persample 1.000+-0.000 median_regret +1.52%+-0.06pp diversity 0.0000+-0.0000 + cvae : stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 1.000+-0.000 median_regret +1.37%+-0.05pp diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 120 plant x draw trials): cvae wins 0/0 discordant, p=1 + +-- K=8 -- + diffusion : stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 1.000+-0.000 median_regret +1.11%+-0.09pp diversity 0.0142+-0.0007 + cvae : stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 1.000+-0.000 median_regret +1.20%+-0.02pp diversity 0.0027+-0.0001 + paired McNemar (diffusion vs cvae, pooled 120 plant x draw trials): cvae wins 0/0 discordant, p=1 + diff --git a/results/ablations/ablation_generative_siso.txt b/results/ablations/ablation_generative_siso.txt new file mode 100644 index 0000000000000000000000000000000000000000..45c99d23de7bc9db0c594d3c90369a39f4949222 --- /dev/null +++ b/results/ablations/ablation_generative_siso.txt @@ -0,0 +1,18 @@ +=== A3 GENERATIVE BASELINE — SISO (N=60 plants, plant_seed=100, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=21,445,653 epochs=1200 time_to_best=7940.6s; cvae params=1,108,021 epochs=60 time_to_best=33.9s; regressor params=804,373 epochs=40 time=not recorded + +-- K=1 -- + diffusion : plant_stabilisable 0.667+-0.033 persample 0.667+-0.033 median_best_J 6.713+-3.633 diversity 0.0000+-0.0000 + cvae : plant_stabilisable 0.750+-0.050 persample 0.750+-0.050 median_best_J 5.167+-2.512 diversity 0.0000+-0.0000 + regressor : plant_stabilisable 0.750+-0.044 persample 0.750+-0.044 median_best_J 2.403+-0.141 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 18/21 discordant, p=0.00149 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 19/23 discordant, p=0.0026 + +-- K=8 -- + diffusion : plant_stabilisable 0.800+-0.050 persample 0.640+-0.024 median_best_J 1.363+-0.120 diversity 0.1844+-0.0076 + cvae : plant_stabilisable 0.767+-0.033 persample 0.751+-0.045 median_best_J 0.748+-0.113 diversity 0.0034+-0.0001 + regressor : plant_stabilisable 0.750+-0.044 persample 0.750+-0.044 median_best_J 2.403+-0.141 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 0/6 discordant, p=0.0312 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 0/9 discordant, p=0.00391 + diff --git a/results/ablations/ablation_generative_siso_pooled.txt b/results/ablations/ablation_generative_siso_pooled.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c6e622469c9309fa3e8af2050b2f6d78cccd15a --- /dev/null +++ b/results/ablations/ablation_generative_siso_pooled.txt @@ -0,0 +1,21 @@ +=== SISO POOLED: 5 held-out plant draws (seeds 100,200,300,400,500), n=300 plants x 3 sampler draws = 900 trials/arm/K === +plant seeds pooled: [100, 200, 300, 400, 500] + +-- K=1 -- + diffusion : coverage 0.628 [0.597, 0.660] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.667, seed200=0.644, seed300=0.672, seed400=0.622, seed500=0.533 + cvae : coverage 0.726 [0.697, 0.756] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.750, seed200=0.739, seed300=0.811, seed400=0.694, seed500=0.633 + regressor : coverage 0.729 [0.700, 0.759] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.750, seed200=0.750, seed300=0.806, seed400=0.700, seed500=0.639 + diffusion - cvae: coverage diff -0.098 [-0.121, -0.076] McNemar b(cvae-only-success)=103 c(diffusion-only-success)=15 n_disc=118 p=2.54e-17 + diffusion - regressor: coverage diff -0.101 [-0.123, -0.079] McNemar b(regressor-only-success)=103 c(diffusion-only-success)=12 n_disc=115 p=3.35e-19 + +-- K=8 -- + diffusion : coverage 0.773 [0.746, 0.801] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.800, seed200=0.756, seed300=0.839, seed400=0.761, seed500=0.711 + cvae : coverage 0.748 [0.719, 0.777] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.767, seed200=0.756, seed300=0.817, seed400=0.739, seed500=0.661 + regressor : coverage 0.729 [0.700, 0.759] (bootstrap 95% CI, n=900 pooled trials) per-draw: seed100=0.750, seed200=0.750, seed300=0.806, seed400=0.700, seed500=0.639 + diffusion - cvae: coverage diff +0.026 [+0.013, +0.039] McNemar b(cvae-only-success)=5 c(diffusion-only-success)=28 n_disc=33 p=6.62e-05 + diffusion - regressor: coverage diff +0.044 [+0.030, +0.060] McNemar b(regressor-only-success)=4 c(diffusion-only-success)=44 n_disc=48 p=1.51e-09 + +-- K=1 -> K=8 coverage GAIN per arm (paired bootstrap CI; the mechanism claim) -- + diffusion : gain +0.146 [+0.122, +0.170] + cvae : gain +0.022 [+0.013, +0.032] + regressor : gain +0.000 [+0.000, +0.000] diff --git a/results/ablations/ablation_generative_siso_seed200.txt b/results/ablations/ablation_generative_siso_seed200.txt new file mode 100644 index 0000000000000000000000000000000000000000..43321bcacfdc692f3ad63f41de967f85e04d841d --- /dev/null +++ b/results/ablations/ablation_generative_siso_seed200.txt @@ -0,0 +1,18 @@ +=== A3 GENERATIVE BASELINE — SISO (N=60 plants, plant_seed=200, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=21,445,653 epochs=1200 time_to_best=7940.6s; cvae params=1,108,021 epochs=60 time_to_best=33.9s; regressor params=804,373 epochs=40 time=not recorded + +-- K=1 -- + diffusion : plant_stabilisable 0.644+-0.067 persample 0.644+-0.067 median_best_J 11.448+-5.757 diversity 0.0000+-0.0000 + cvae : plant_stabilisable 0.739+-0.025 persample 0.739+-0.025 median_best_J 5.993+-2.409 diversity 0.0000+-0.0000 + regressor : plant_stabilisable 0.750+-0.029 persample 0.750+-0.029 median_best_J 2.380+-0.277 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 18/19 discordant, p=7.63e-05 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 19/19 discordant, p=3.81e-06 + +-- K=8 -- + diffusion : plant_stabilisable 0.756+-0.069 persample 0.615+-0.017 median_best_J 1.796+-0.349 diversity 0.1773+-0.0099 + cvae : plant_stabilisable 0.756+-0.038 persample 0.733+-0.030 median_best_J 0.770+-0.091 diversity 0.0034+-0.0001 + regressor : plant_stabilisable 0.750+-0.029 persample 0.750+-0.029 median_best_J 2.380+-0.277 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 2/4 discordant, p=1 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 2/5 discordant, p=1 + diff --git a/results/ablations/ablation_generative_siso_seed300.txt b/results/ablations/ablation_generative_siso_seed300.txt new file mode 100644 index 0000000000000000000000000000000000000000..cfa22052d79d744e2912f85b82f58e5a72e474ca --- /dev/null +++ b/results/ablations/ablation_generative_siso_seed300.txt @@ -0,0 +1,18 @@ +=== A3 GENERATIVE BASELINE — SISO (N=60 plants, plant_seed=300, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=21,445,653 epochs=1200 time_to_best=7940.6s; cvae params=1,108,021 epochs=60 time_to_best=33.9s; regressor params=804,373 epochs=40 time=not recorded + +-- K=1 -- + diffusion : plant_stabilisable 0.672+-0.054 persample 0.672+-0.054 median_best_J 7.642+-3.111 diversity 0.0000+-0.0000 + cvae : plant_stabilisable 0.811+-0.035 persample 0.811+-0.035 median_best_J 5.722+-3.110 diversity 0.0000+-0.0000 + regressor : plant_stabilisable 0.806+-0.025 persample 0.806+-0.025 median_best_J 2.596+-0.253 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 26/27 discordant, p=4.17e-07 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 25/26 discordant, p=8.05e-07 + +-- K=8 -- + diffusion : plant_stabilisable 0.839+-0.025 persample 0.681+-0.016 median_best_J 1.486+-0.162 diversity 0.1781+-0.0086 + cvae : plant_stabilisable 0.817+-0.033 persample 0.806+-0.023 median_best_J 0.913+-0.223 diversity 0.0034+-0.0001 + regressor : plant_stabilisable 0.806+-0.025 persample 0.806+-0.025 median_best_J 2.596+-0.253 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 0/4 discordant, p=0.125 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 0/6 discordant, p=0.0312 + diff --git a/results/ablations/ablation_generative_siso_seed400.txt b/results/ablations/ablation_generative_siso_seed400.txt new file mode 100644 index 0000000000000000000000000000000000000000..bfe8c3af69a616206d8c33c4b4b657349863d79c --- /dev/null +++ b/results/ablations/ablation_generative_siso_seed400.txt @@ -0,0 +1,18 @@ +=== A3 GENERATIVE BASELINE — SISO (N=60 plants, plant_seed=400, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=21,445,653 epochs=1200 time_to_best=7940.6s; cvae params=1,108,021 epochs=60 time_to_best=33.9s; regressor params=804,373 epochs=40 time=not recorded + +-- K=1 -- + diffusion : plant_stabilisable 0.622+-0.042 persample 0.622+-0.042 median_best_J 9.790+-5.143 diversity 0.0000+-0.0000 + cvae : plant_stabilisable 0.694+-0.025 persample 0.694+-0.025 median_best_J 4.571+-1.419 diversity 0.0000+-0.0000 + regressor : plant_stabilisable 0.700+-0.044 persample 0.700+-0.044 median_best_J 2.706+-0.383 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 20/27 discordant, p=0.0192 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 20/26 discordant, p=0.00936 + +-- K=8 -- + diffusion : plant_stabilisable 0.761+-0.035 persample 0.608+-0.017 median_best_J 1.968+-0.313 diversity 0.1875+-0.0109 + cvae : plant_stabilisable 0.739+-0.042 persample 0.697+-0.031 median_best_J 0.908+-0.171 diversity 0.0034+-0.0000 + regressor : plant_stabilisable 0.700+-0.044 persample 0.700+-0.044 median_best_J 2.706+-0.383 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 1/6 discordant, p=0.219 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 1/13 discordant, p=0.00342 + diff --git a/results/ablations/ablation_generative_siso_seed500.txt b/results/ablations/ablation_generative_siso_seed500.txt new file mode 100644 index 0000000000000000000000000000000000000000..6eb51e8fa08f567e89371e0c996b05c01f7b457e --- /dev/null +++ b/results/ablations/ablation_generative_siso_seed500.txt @@ -0,0 +1,18 @@ +=== A3 GENERATIVE BASELINE — SISO (N=60 plants, plant_seed=500, 3 draws/K, seeds=[0, 1, 2]) === + +budget: diffusion params=21,445,653 epochs=1200 time_to_best=7940.6s; cvae params=1,108,021 epochs=60 time_to_best=33.9s; regressor params=804,373 epochs=40 time=not recorded + +-- K=1 -- + diffusion : plant_stabilisable 0.533+-0.017 persample 0.533+-0.017 median_best_J 5.136+-2.005 diversity 0.0000+-0.0000 + cvae : plant_stabilisable 0.633+-0.017 persample 0.633+-0.017 median_best_J 5.881+-2.407 diversity 0.0000+-0.0000 + regressor : plant_stabilisable 0.639+-0.025 persample 0.639+-0.025 median_best_J 2.357+-0.177 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 21/24 discordant, p=0.000277 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 20/21 discordant, p=2.1e-05 + +-- K=8 -- + diffusion : plant_stabilisable 0.711+-0.048 persample 0.537+-0.028 median_best_J 1.432+-0.271 diversity 0.1897+-0.0077 + cvae : plant_stabilisable 0.661+-0.019 persample 0.635+-0.018 median_best_J 0.774+-0.171 diversity 0.0034+-0.0001 + regressor : plant_stabilisable 0.639+-0.025 persample 0.639+-0.025 median_best_J 2.357+-0.177 diversity 0.0000+-0.0000 + paired McNemar (diffusion vs cvae, pooled 180 plant x draw trials): cvae wins 2/13 discordant, p=0.0225 + paired McNemar (diffusion vs regressor, pooled 180 plant x draw trials): regressor wins 1/15 discordant, p=0.000977 + diff --git a/results/ablations/ablation_size.txt b/results/ablations/ablation_size.txt new file mode 100644 index 0000000000000000000000000000000000000000..c0730572591b908f033649a7192881243cc5ac72 --- /dev/null +++ b/results/ablations/ablation_size.txt @@ -0,0 +1,18 @@ +=== SIZE ABLATION (pendulum, N=40, K=8, shared cached oracle, eval noise seed=0, 3 training seeds=[42, 43, 44]) === + +- hidden=64 (~1/4 width) + params=255,563 time_to_best=405.9s + median regret vs oracle: +0.68% +/- 0.16pp over seeds [s42=+0.58%, s43=+0.87%, s44=+0.60%] + coverage (mean over seeds): certified>=1/plant 0.992 stable>=1/plant 1.000 per-sample stable 0.999 +- hidden=256 (fair reference, ablation trainer) + params=3,823,883 time_to_best=421.4s + median regret vs oracle: +0.92% +/- 0.22pp over seeds [s42=+1.06%, s43=+0.67%, s44=+1.03%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 +- hidden=512 (~2x width) + params=15,118,859 time_to_best=441.1s + median regret vs oracle: +0.86% +/- 0.38pp over seeds [s42=+0.98%, s43=+0.43%, s44=+1.16%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 +- hidden=256 (DEPLOYED checkpoint -- pre-existing train_pendulum.py run, not part of the 3-seed sweep) + params=3,823,883 time_to_best=232.0s + median regret vs oracle: +1.15% +/- 0.00pp over seeds [ssingle=+1.15%] + coverage (mean over seeds): certified>=1/plant 1.000 stable>=1/plant 1.000 per-sample stable 1.000 diff --git a/results/ablations/ablation_steps.txt b/results/ablations/ablation_steps.txt new file mode 100644 index 0000000000000000000000000000000000000000..2f0a99e24d991ec737df8bee98488d3335e87a8c --- /dev/null +++ b/results/ablations/ablation_steps.txt @@ -0,0 +1,14 @@ +=== A4 STEP-0: DDIM@500 vs DDPM@500 quality check (3 draws each, seeds=[0, 1, 2]) === + +DDPM@500 : stable 1.000+-0.000 cert 1.000+-0.000 regret +1.11%+-0.09pp gen 1.747+-0.400 ms/ctrl +DDIM@500 : stable 1.000+-0.000 cert 1.000+-0.000 regret +1.05%+-0.05pp gen 2.218+-0.743 ms/ctrl +VERDICT: DDIM@500 MATCHES DDPM@500 within noise (|stable diff|=0.000, |regret diff|=0.06pp) + +=== A4 STEPS SWEEP (DDIM, eta=0, N=40, K=8, 3 draws/step-count, seeds=[0, 1, 2]) === + +steps= 5 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 0.995+-0.004 median_regret +0.96%+-0.14pp wall 0.0304+-0.0069 ms/ctrl +steps= 10 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 0.999+-0.002 median_regret +1.06%+-0.05pp wall 0.0362+-0.0095 ms/ctrl +steps= 20 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 0.999+-0.002 median_regret +1.06%+-0.05pp wall 0.0982+-0.0376 ms/ctrl +steps= 50 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 0.999+-0.002 median_regret +1.08%+-0.04pp wall 0.2618+-0.0955 ms/ctrl +steps= 100 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 1.000+-0.000 median_regret +1.08%+-0.05pp wall 0.4116+-0.0402 ms/ctrl +steps= 500 stable>=1 1.000+-0.000 cert>=1 1.000+-0.000 persample 0.999+-0.002 median_regret +1.05%+-0.05pp wall 1.8847+-0.3124 ms/ctrl diff --git a/results/ablations/ablation_steps_mimo.txt b/results/ablations/ablation_steps_mimo.txt new file mode 100644 index 0000000000000000000000000000000000000000..68691d094e5d56ad287cf32aa5c88721bf48d94b --- /dev/null +++ b/results/ablations/ablation_steps_mimo.txt @@ -0,0 +1,8 @@ +=== A4 MIMO bonus: DDPM@500 anchor + DDIM steps sweep (N=60, K=6, 3 draws, seeds=[0, 1, 2]) === + +DDPM@500 anchor: per-sample-stable 0.975+-0.006 median_regret_vs_phi0 +0.5%+-0.1 gen 3.952+-1.367 ms/ctrl + +steps= 5 per-sample-stable 0.975+-0.007 has_stable>=1 1.000+-0.000 within_sector 0.890+-0.018 median_regret_vs_phi0 +1.2%+-0.5 gen 0.0357+-0.0205 ms/ctrl +steps= 20 per-sample-stable 0.972+-0.007 has_stable>=1 0.994+-0.010 within_sector 0.840+-0.013 median_regret_vs_phi0 +0.6%+-0.5 gen 0.2460+-0.2445 ms/ctrl +steps= 100 per-sample-stable 0.964+-0.007 has_stable>=1 0.994+-0.010 within_sector 0.789+-0.005 median_regret_vs_phi0 +0.4%+-0.2 gen 0.4678+-0.0237 ms/ctrl +steps= 500 per-sample-stable 0.968+-0.018 has_stable>=1 0.994+-0.010 within_sector 0.778+-0.007 median_regret_vs_phi0 +0.3%+-0.2 gen 2.6111+-0.1449 ms/ctrl diff --git a/results/ablations/pendulum_data_anomaly.txt b/results/ablations/pendulum_data_anomaly.txt new file mode 100644 index 0000000000000000000000000000000000000000..01db0ba7a788df085bc0961235cec700c290c98c --- /dev/null +++ b/results/ablations/pendulum_data_anomaly.txt @@ -0,0 +1,24 @@ +=== PENDULUM data-size anomaly diagnostics === +held-out: seed123 (N=40) + seed321 (N=40), pooled N=80 plants; sampler draws (0, 1, 2); K=1 = first sample of each draw; K=8 = best of 8. +regret % vs strong per-plant oracle; failures (unstable) counted as +inf in the median, never dropped. +CI = 95% bootstrap over plants (5000) of the median pooled over (training seed x draw). + +TARGET native regret (random 40 training targets, seed 2026, on own plants, K=1): median +1.41 [+1.27,+1.51] p90 +3.59 max +4.09 fail 0.000 + +config seeds K1 median [CI] K1 p90 K1 fail K8 median [CI] sd_seed K1 sd_seed K8 d_min d_nnplant +f0.25_e800 3 +1.16 [+0.99,+1.40] 4.18 0.000 +0.28 [+0.21,+0.37] 0.25 0.22 0.015 0.063 +f0.50_e800 3 +1.48 [+1.33,+1.63] 3.06 0.000 +0.90 [+0.78,+1.04] 0.27 0.22 0.005 0.044 +f1.00_e800 3 +1.32 [+1.15,+1.47] 3.09 0.000 +0.91 [+0.79,+1.03] 0.23 0.22 0.004 0.035 +f0.25_e3200 2 +1.43 [+1.33,+1.58] 3.14 0.000 +1.07 [+0.95,+1.18] 0.03 0.08 0.004 0.049 +f1.00_e200 2 +1.43 [+1.28,+1.63] 3.42 0.000 +0.52 [+0.45,+0.61] 0.28 0.31 0.012 0.046 + +per-plant paired Wilcoxon (plant = mean over seed x draw of regret; same 80 plants), two-sided: + f0.25_e800 vs f1.00_e800: K1 med diff +0.05pp p=0.146 (n=80) | K8 med diff -0.49pp p=8.46e-14 (n=80) + f0.25_e3200 vs f0.25_e800: K1 med diff +0.12pp p=0.77 (n=80) | K8 med diff +0.71pp p=3.38e-14 (n=80) + f0.25_e3200 vs f1.00_e800: K1 med diff +0.12pp p=1.59e-07 (n=80) | K8 med diff +0.12pp p=5.96e-09 (n=80) + f1.00_e200 vs f1.00_e800: K1 med diff +0.15pp p=0.00533 (n=80) | K8 med diff -0.34pp p=3.5e-11 (n=80) + f1.00_e200 vs f0.25_e800: K1 med diff +0.15pp p=0.181 (n=80) | K8 med diff +0.18pp p=1.7e-06 (n=80) + f0.25_e800 vs f0.50_e800: K1 med diff -0.11pp p=0.996 (n=80) | K8 med diff -0.48pp p=1.57e-13 (n=80) + +Model K=1 vs native target regret (unpaired, different plants): compare K1 median column to the TARGET line above. +d_min = distance from a generated (clipped, normalized) controller to the nearest vector in that model's own training subset; d_nnplant = distance to the target of the nearest-condition training plant. diff --git a/results/furuta_v4/ablation_interp_mlpA_raw_mlpA_ema9995_mlpA_ema999_mlp192k_ema999_mlp3.txt b/results/furuta_v4/ablation_interp_mlpA_raw_mlpA_ema9995_mlpA_ema999_mlp192k_ema999_mlp3.txt new file mode 100644 index 0000000000000000000000000000000000000000..8677bcaf4a9e21cb33ccf62588ebf5b1b8577640 --- /dev/null +++ b/results/furuta_v4/ablation_interp_mlpA_raw_mlpA_ema9995_mlpA_ema999_mlp192k_ema999_mlp3.txt @@ -0,0 +1,11 @@ +model K=1 (draw4242/draw9001, pooled, paired diff to nearest+P) // K=4 +interp K=1: 60.0/61.7 pooled 60.8 vs retr 54.2 | diff +6.7 [-5.0,+17.5] p=0.2979 K=4: 70.0/71.7 pooled 70.8 vs retr 82.5 | diff -11.7 [-20.0,-3.3] p=0.0129 +mlpA_raw K=1: 53.9/57.2 pooled 55.6 vs retr 54.2 | diff +1.4 [-8.3,+10.8] p=0.8250 K=4: 83.9/90.6 pooled 87.2 vs retr 82.5 | diff +4.7 [-0.8,+10.6] p=0.1459 +mlpA_ema9995 K=1: 53.9/56.7 pooled 55.3 vs retr 54.2 | diff +1.1 [-8.6,+10.6] p=0.8675 K=4: 83.9/90.6 pooled 87.2 vs retr 82.5 | diff +4.7 [-0.8,+10.8] p=0.1433 +mlpA_ema999 K=1: 57.2/64.4 pooled 60.8 vs retr 54.2 | diff +6.7 [-3.3,+16.4] p=0.2075 K=4: 85.6/90.0 pooled 87.8 vs retr 82.5 | diff +5.3 [-0.6,+11.4] p=0.1124 +mlp192k_ema999 K=1: 60.0/73.3 pooled 66.7 vs retr 54.2 | diff +12.5 [+3.1,+21.7] p=0.0146 * K=4: 85.0/90.6 pooled 87.8 vs retr 82.5 | diff +5.3 [-0.3,+11.1] p=0.0946 +mlp384k_ema9995 K=1: 66.1/80.0 pooled 73.1 vs retr 54.2 | diff +18.9 [+9.2,+28.3] p=0.0005 * K=4: 87.8/91.1 pooled 89.4 vs retr 82.5 | diff +6.9 [+1.7,+12.5] p=0.0183 * +mlp384k_ema999 K=1: 66.1/80.6 pooled 73.3 vs retr 54.2 | diff +19.2 [+8.9,+28.9] p=0.0002 * K=4: 87.8/91.1 pooled 89.4 vs retr 82.5 | diff +6.9 [+1.7,+12.5] p=0.0192 * +unetS_ema999 K=1: 88.3/92.2 pooled 90.3 vs retr 54.2 | diff +36.1 [+27.5,+45.3] p=0.0000 * K=4: 95.0/95.0 pooled 95.0 vs retr 82.5 | diff +12.5 [+7.5,+18.3] p=0.0002 * +unetA_ema9995 K=1: 90.6/93.3 pooled 91.9 vs retr 54.2 | diff +37.8 [+29.2,+46.7] p=0.0000 * K=4: 92.8/93.9 pooled 93.3 vs retr 82.5 | diff +10.8 [+5.0,+16.9] p=0.0004 * +unetA_ema999 K=1: 90.6/93.3 pooled 91.9 vs retr 54.2 | diff +37.8 [+28.9,+46.7] p=0.0000 * K=4: 92.8/93.9 pooled 93.3 vs retr 82.5 | diff +10.8 [+5.3,+16.9] p=0.0005 * diff --git a/results/furuta_v4/ablation_mlpB256_ema999_mlpXD_ema999.txt b/results/furuta_v4/ablation_mlpB256_ema999_mlpXD_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..778072dc14dfca9ad1228acdb025a5e054152bd2 --- /dev/null +++ b/results/furuta_v4/ablation_mlpB256_ema999_mlpXD_ema999.txt @@ -0,0 +1,3 @@ +model K=1 (draw4242/draw9001, pooled, paired diff to nearest+P) // K=4 +mlpB256_ema999 K=1: 74.4/75.0 pooled 74.7 vs retr 54.2 | diff +20.6 [+10.3,+30.8] p=0.0002 * K=4: 88.3/91.7 pooled 90.0 vs retr 82.5 | diff +7.5 [+2.2,+13.1] p=0.0106 * +mlpXD_ema999 K=1: 75.6/77.8 pooled 76.7 vs retr 67.5 | diff +9.2 [+0.6,+18.1] p=0.0499 * K=4: 89.4/93.3 pooled 91.4 vs retr 90.0 | diff +1.4 [-2.2,+5.6] p=0.5889 diff --git a/results/furuta_v4/ablation_table.txt b/results/furuta_v4/ablation_table.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b7c7ea077755a2b552aafeb26f326894d783515 --- /dev/null +++ b/results/furuta_v4/ablation_table.txt @@ -0,0 +1,8 @@ +model K=1 (draw4242/draw9001, pooled, paired diff to nearest+P) // K=4 +unetAseed1_ema999 K=1: 92.8/93.9 pooled 93.3 vs retr 54.2 | diff +39.2 [+30.0,+47.8] p=0.0000 * K=4: 93.3/94.4 pooled 93.9 vs retr 82.5 | diff +11.4 [+5.6,+17.5] p=0.0006 * +unetAseed2_ema999 K=1: 90.6/93.3 pooled 91.9 vs retr 54.2 | diff +37.8 [+29.2,+46.7] p=0.0000 * K=4: 91.7/93.3 pooled 92.5 vs retr 82.5 | diff +10.0 [+4.2,+15.8] p=0.0022 * +unetA_ema999 K=1: 90.6/93.3 pooled 91.9 vs retr 54.2 | diff +37.8 [+28.6,+46.9] p=0.0000 * K=4: 92.8/93.9 pooled 93.3 vs retr 82.5 | diff +10.8 [+5.0,+17.2] p=0.0006 * +unetS_ema999 K=1: 88.3/92.2 pooled 90.3 vs retr 54.2 | diff +36.1 [+27.2,+44.7] p=0.0000 * K=4: 95.0/95.0 pooled 95.0 vs retr 82.5 | diff +12.5 [+6.7,+18.3] p=0.0001 * +unetSXD_ema999 K=1: 90.0/91.1 pooled 90.6 vs retr 67.5 | diff +23.1 [+15.0,+31.4] p=0.0000 * K=4: 95.0/95.6 pooled 95.3 vs retr 90.0 | diff +5.3 [+0.8,+10.0] p=0.0380 * +mlpB256_ema999 K=1: 74.4/75.0 pooled 74.7 vs retr 54.2 | diff +20.6 [+10.3,+30.3] p=0.0002 * K=4: 88.3/91.7 pooled 90.0 vs retr 82.5 | diff +7.5 [+1.9,+13.3] p=0.0113 * +mlpXD_ema999 K=1: 75.6/77.8 pooled 76.7 vs retr 67.5 | diff +9.2 [+0.3,+18.1] p=0.0500 K=4: 89.4/93.3 pooled 91.4 vs retr 90.0 | diff +1.4 [-2.2,+5.6] p=0.5806 diff --git a/results/furuta_v4/ablation_unetSXD_ema999.txt b/results/furuta_v4/ablation_unetSXD_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..82d54c3cd3118653a59f9f600e1ab3595475af4e --- /dev/null +++ b/results/furuta_v4/ablation_unetSXD_ema999.txt @@ -0,0 +1,2 @@ +model K=1 (draw4242/draw9001, pooled, paired diff to nearest+P) // K=4 +unetSXD_ema999 K=1: 90.0/91.1 pooled 90.6 vs retr 67.5 | diff +23.1 [+15.3,+31.4] p=0.0000 * K=4: 95.0/95.6 pooled 95.3 vs retr 90.0 | diff +5.3 [+0.8,+10.0] p=0.0391 * diff --git a/results/furuta_v4/budget_curves_ddim5.txt b/results/furuta_v4/budget_curves_ddim5.txt new file mode 100644 index 0000000000000000000000000000000000000000..74d365d89ae3c2e02d408679c8d4e50bdfa01ce5 --- /dev/null +++ b/results/furuta_v4/budget_curves_ddim5.txt @@ -0,0 +1,27 @@ +# success (%) vs per-plant single-thread wall-clock budget; verify times as measured, costs from budget_timing_cpu.txt; refine n=200 +# cost model: sample1 0.02s sample4 0.04s regfwd1 3.2ms refine(40 steps) 0.70s (flat in batch<=4) +method 0.1s 0.25s 0.5s 1s 2s 3s 5s 10s 60s best config at each budget +retrieval alone 0.0 0.0 48.3 69.2 77.5 83.3 90.0 93.3 93.3 - | - | S=16,m=1 | S=16,m=3 | S=16,m=6 | S=16,m=6 | S=16,m=7 | S=16,m=11 | S=16,m=11 +retrieval + refine 0.0 0.0 48.3 83.3 87.5 93.3 97.5 97.5 97.5 - | - | S=4,m=1 | S=4,m=1,I=20 | S=4,m=2,I=40 | S=4,m=2,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 +generated (U-Net, DDIM-5) alone 0.0 0.0 80.3 81.7 82.7 92.9 93.8 94.1 94.3 - | - | S=1,m=1 | S=4,m=3 | S=8,m=5 | S=8,m=8 | S=16,m=11 | S=16,m=11 | S=16,m=13 +generated (U-Net, DDIM-5) + refine 0.0 0.0 80.3 85.6 87.4 97.2 98.0 98.1 98.1 - | - | S=1,m=1 | S=1,m=1,I=20 | S=4,m=2,I=40 | S=4,m=2,I=40 | S=4,m=3,I=40 | S=4,m=4,I=40 | S=4,m=4,I=40 +regression twin (deterministic) alone 0.0 0.8 76.9 76.9 76.9 85.6 85.6 85.6 85.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 +regression twin (deterministic) + refine 0.0 0.8 76.9 83.3 84.7 93.3 95.6 95.6 95.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1,I=20 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 + +# paired difference A - B in pp at each budget [95% CI] p (best config per method per budget; 120 plants) +generated (U-Net, DDIM-5) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +31.9 [+23.1,+41.0] p=0.000 +12.5 [ +5.1,+19.8] p=0.001 +5.2 [ -1.9,+12.3] p=0.165 +9.5 [ +4.3,+15.3] p=0.001 +3.8 [ +0.5, +7.9] p=0.047 +0.7 [ -2.4, +4.3] p=0.719 +0.9 [ -2.3, +4.4] p=0.622 +generated (U-Net, DDIM-5) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +31.9 [+22.6,+41.3] p=0.000 +2.3 [ -4.4, +9.3] p=0.531 -0.1 [ -6.6, +6.8] p=1.000 +3.9 [ +0.4, +8.0] p=0.046 +0.5 [ -0.6, +2.3] p=1.000 +0.6 [ -0.4, +2.5] p=1.000 +0.6 [ -0.4, +2.5] p=1.000 +generated (U-Net, DDIM-5) alone - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +31.9 [+23.0,+40.6] p=0.000 -1.7 [ -8.1, +5.3] p=0.658 -4.8 [-12.1, +2.1] p=0.198 -0.5 [ -4.9, +3.8] p=0.879 -3.7 [ -7.9, +0.0] p=0.086 -3.4 [ -7.5, +0.1] p=0.091 -3.2 [ -7.1, +0.4] p=0.103 +generated (U-Net, DDIM-5) + refine - generated (U-Net, DDIM-5) alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +4.0 [ +1.7, +6.8] p=0.001 +4.7 [ +1.8, +8.1] p=0.002 +4.4 [ +1.6, +7.8] p=0.002 +4.2 [ +1.4, +7.4] p=0.016 +4.1 [ +1.1, +7.6] p=0.016 +3.9 [ +1.1, +7.4] p=0.015 +regression twin (deterministic) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +28.6 [+19.4,+38.1] p=0.000 +7.8 [ -0.0,+15.3] p=0.058 -0.6 [ -8.3, +7.2] p=0.943 +2.2 [ -3.6, +8.1] p=0.523 -4.4 [ -9.2, -0.0] p=0.085 -7.8 [-12.8, -3.1] p=0.002 -7.8 [-13.1, -3.1] p=0.002 +regression twin (deterministic) alone - generated (U-Net, DDIM-5) alone + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 -3.3 [ -7.6, +0.6] p=0.131 -4.7 [ -9.4, -0.4] p=0.042 -5.7 [-10.5, -1.1] p=0.018 -7.3 [-11.7, -3.3] p=0.000 -8.2 [-12.9, -4.3] p=0.000 -8.5 [-13.1, -4.4] p=0.000 -8.7 [-13.2, -4.5] p=0.000 +regression twin (deterministic) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +28.6 [+19.2,+38.1] p=0.000 -0.0 [ -7.2, +7.2] p=1.000 -2.8 [ -9.4, +4.2] p=0.479 -0.0 [ -3.9, +3.9] p=1.000 -1.9 [ -5.3, +1.1] p=0.381 -1.9 [ -5.6, +1.1] p=0.366 -1.9 [ -5.3, +1.1] p=0.371 +regression twin (deterministic) + refine - generated (U-Net, DDIM-5) + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 -3.3 [ -7.6, +0.7] p=0.132 -2.3 [ -6.3, +1.6] p=0.267 -2.7 [ -6.8, +1.3] p=0.215 -3.9 [ -7.1, -1.1] p=0.014 -2.4 [ -5.3, -0.3] p=0.126 -2.6 [ -5.6, -0.4] p=0.127 -2.6 [ -5.6, -0.4] p=0.124 diff --git a/results/furuta_v4/budget_curves_ddpm500.txt b/results/furuta_v4/budget_curves_ddpm500.txt new file mode 100644 index 0000000000000000000000000000000000000000..e5794957d08fa8cc3af63e28a4747aa87da8b414 --- /dev/null +++ b/results/furuta_v4/budget_curves_ddpm500.txt @@ -0,0 +1,27 @@ +# success (%) vs per-plant single-thread wall-clock budget; verify times as measured, costs from budget_timing_cpu.txt; refine n=200 +# cost model: sample1 1.56s sample4 3.51s regfwd1 3.2ms refine(40 steps) 0.70s (flat in batch<=4) +method 0.1s 0.25s 0.5s 1s 2s 3s 5s 10s 60s best config at each budget +retrieval alone 0.0 0.0 48.3 69.2 77.5 83.3 90.0 93.3 93.3 - | - | S=16,m=1 | S=16,m=3 | S=16,m=6 | S=16,m=6 | S=16,m=7 | S=16,m=11 | S=16,m=11 +retrieval + refine 0.0 0.0 48.3 83.3 87.5 93.3 97.5 97.5 97.5 - | - | S=4,m=1 | S=4,m=1,I=20 | S=4,m=2,I=40 | S=4,m=2,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 +generated (U-Net, DDPM-500) alone 0.0 0.0 0.0 0.0 72.1 82.5 85.8 93.0 94.2 - | - | - | - | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=4,m=4 | S=16,m=16 +generated (U-Net, DDPM-500) + refine 0.0 0.0 0.0 0.0 72.1 86.5 91.1 98.0 98.4 - | - | - | - | S=1,m=1 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=4,m=3,I=40 | S=4,m=4,I=40 +regression twin (deterministic) alone 0.0 0.8 76.9 76.9 76.9 85.6 85.6 85.6 85.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 +regression twin (deterministic) + refine 0.0 0.8 76.9 83.3 84.7 93.3 95.6 95.6 95.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1,I=20 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 + +# paired difference A - B in pp at each budget [95% CI] p (best config per method per budget; 120 plants) +generated (U-Net, DDPM-500) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-39.2] p=0.000 -69.2 [-77.5,-60.8] p=0.000 -5.4 [-12.1, +1.4] p=0.135 -0.8 [ -6.9, +5.6] p=0.821 -4.2 [ -8.8, +0.6] p=0.100 -0.4 [ -3.6, +3.1] p=0.910 +0.8 [ -2.5, +4.4] p=0.811 +generated (U-Net, DDPM-500) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-39.2] p=0.000 -83.3 [-89.2,-76.7] p=0.000 -15.4 [-22.1, -8.5] p=0.000 -6.9 [-12.5, -1.0] p=0.024 -6.4 [-10.4, -2.5] p=0.001 +0.5 [ -1.4, +2.8] p=0.749 +0.9 [ -0.9, +3.3] p=0.499 +generated (U-Net, DDPM-500) alone - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-40.0] p=0.000 -83.3 [-90.0,-75.8] p=0.000 -15.4 [-22.4, -8.6] p=0.000 -10.8 [-16.8, -5.2] p=0.000 -11.7 [-17.2, -6.6] p=0.000 -4.5 [ -8.8, -0.6] p=0.045 -3.3 [ -7.4, +0.2] p=0.140 +generated (U-Net, DDPM-500) + refine - generated (U-Net, DDPM-500) alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +4.0 [ +1.5, +6.9] p=0.002 +5.3 [ +1.9, +8.9] p=0.002 +5.0 [ +1.7, +8.9] p=0.008 +4.3 [ +1.2, +8.0] p=0.014 +regression twin (deterministic) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +28.6 [+19.4,+38.1] p=0.000 +7.8 [ -0.0,+15.3] p=0.058 -0.6 [ -8.3, +7.2] p=0.943 +2.2 [ -3.6, +8.1] p=0.523 -4.4 [ -9.2, -0.0] p=0.085 -7.8 [-12.8, -3.1] p=0.002 -7.8 [-13.1, -3.1] p=0.002 +regression twin (deterministic) alone - generated (U-Net, DDPM-500) alone + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +76.9 [+70.0,+83.6] p=0.000 +76.9 [+70.3,+83.3] p=0.000 +4.8 [ +0.3, +9.1] p=0.041 +3.1 [ -2.4, +8.8] p=0.313 -0.3 [ -5.1, +4.4] p=0.940 -7.4 [-11.9, -3.5] p=0.000 -8.6 [-13.4, -4.4] p=0.000 +regression twin (deterministic) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +28.6 [+19.2,+38.1] p=0.000 -0.0 [ -7.2, +7.2] p=1.000 -2.8 [ -9.4, +4.2] p=0.479 -0.0 [ -3.9, +3.9] p=1.000 -1.9 [ -5.3, +1.1] p=0.381 -1.9 [ -5.6, +1.1] p=0.366 -1.9 [ -5.3, +1.1] p=0.371 +regression twin (deterministic) + refine - generated (U-Net, DDPM-500) + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.8 [ +0.0, +2.5] p=1.000 +76.9 [+70.3,+83.6] p=0.000 +83.3 [+77.5,+88.9] p=0.000 +12.6 [ +8.1,+17.1] p=0.000 +6.9 [ +1.9,+12.4] p=0.010 +4.4 [ +0.6, +8.4] p=0.033 -2.4 [ -5.2, -0.2] p=0.066 -2.9 [ -5.6, -0.6] p=0.062 diff --git a/results/furuta_v4/budget_curves_ddpm500_gpusample.txt b/results/furuta_v4/budget_curves_ddpm500_gpusample.txt new file mode 100644 index 0000000000000000000000000000000000000000..de2eb336477b7034cfe033c7915d7d45e820f530 --- /dev/null +++ b/results/furuta_v4/budget_curves_ddpm500_gpusample.txt @@ -0,0 +1,27 @@ +# success (%) vs per-plant single-thread wall-clock budget; verify times as measured, costs from budget_timing_cpu.txt; refine n=200 +# cost model: sample1 0.99s sample4 0.94s regfwd1 2.1ms refine(40 steps) 0.70s (flat in batch<=4) +method 0.1s 0.25s 0.5s 1s 2s 3s 5s 10s 60s best config at each budget +retrieval alone 0.0 0.0 48.3 69.2 77.5 83.3 90.0 93.3 93.3 - | - | S=16,m=1 | S=16,m=3 | S=16,m=6 | S=16,m=6 | S=16,m=7 | S=16,m=11 | S=16,m=11 +retrieval + refine 0.0 0.0 48.3 83.3 87.5 93.3 97.5 97.5 97.5 - | - | S=4,m=1 | S=4,m=1,I=20 | S=4,m=2,I=40 | S=4,m=2,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 | S=4,m=3,I=40 +generated (U-Net, DDPM-500) alone 0.0 0.0 0.0 0.0 83.1 83.2 92.9 94.0 94.2 - | - | - | - | S=4,m=2 | S=4,m=4 | S=4,m=4 | S=16,m=16 | S=16,m=16 +generated (U-Net, DDPM-500) + refine 0.0 0.0 0.0 0.0 85.0 87.4 97.3 98.4 98.4 - | - | - | - | S=4,m=1,I=10 | S=1,m=1,I=40 | S=4,m=1,I=40 | S=4,m=4,I=40 | S=4,m=4,I=40 +regression twin (deterministic) alone 0.0 1.1 76.9 76.9 76.9 85.6 85.6 85.6 85.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 | S=1,m=1 +regression twin (deterministic) + refine 0.0 1.1 76.9 83.3 84.7 93.3 95.6 95.6 95.6 - | S=1,m=1 | S=1,m=1 | S=1,m=1,I=20 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 | S=1,m=1,I=40 + +# paired difference A - B in pp at each budget [95% CI] p (best config per method per budget; 120 plants) +generated (U-Net, DDPM-500) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-39.2] p=0.000 -69.2 [-77.5,-60.8] p=0.000 +5.6 [ -1.6,+12.8] p=0.141 -0.1 [ -6.4, +6.4] p=1.000 +2.9 [ -0.9, +7.2] p=0.195 +0.6 [ -2.4, +4.0] p=0.811 +0.8 [ -2.5, +4.4] p=0.811 +generated (U-Net, DDPM-500) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-39.2] p=0.000 -83.3 [-89.2,-76.7] p=0.000 -2.5 [ -9.4, +4.7] p=0.515 -5.9 [-11.7, -0.1] p=0.054 -0.2 [ -2.8, +2.4] p=1.000 +0.9 [ -0.9, +3.3] p=0.500 +0.9 [ -0.9, +3.3] p=0.499 +generated (U-Net, DDPM-500) alone - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 -48.3 [-57.5,-40.0] p=0.000 -83.3 [-90.0,-75.8] p=0.000 -4.4 [-11.8, +2.5] p=0.246 -10.1 [-16.2, -4.4] p=0.001 -4.6 [ -9.2, -0.5] p=0.041 -3.5 [ -7.6, +0.1] p=0.125 -3.3 [ -7.4, +0.2] p=0.140 +generated (U-Net, DDPM-500) + refine - generated (U-Net, DDPM-500) alone + +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +0.0 [ +0.0, +0.0] p=1.000 +1.9 [ +0.2, +4.3] p=0.124 +4.2 [ +1.3, +7.8] p=0.016 +4.4 [ +1.5, +7.9] p=0.016 +4.4 [ +1.2, +8.3] p=0.016 +4.3 [ +1.2, +8.0] p=0.014 +regression twin (deterministic) alone - retrieval alone + +0.0 [ +0.0, +0.0] p=1.000 +1.1 [ +0.0, +3.1] p=0.502 +28.6 [+19.4,+38.1] p=0.000 +7.8 [ -0.0,+15.3] p=0.058 -0.6 [ -8.3, +7.2] p=0.943 +2.2 [ -3.6, +8.1] p=0.523 -4.4 [ -9.2, -0.0] p=0.085 -7.8 [-12.8, -3.1] p=0.002 -7.8 [-13.1, -3.1] p=0.002 +regression twin (deterministic) alone - generated (U-Net, DDPM-500) alone + +0.0 [ +0.0, +0.0] p=1.000 +1.1 [ +0.0, +3.1] p=0.502 +76.9 [+70.0,+83.6] p=0.000 +76.9 [+70.3,+83.3] p=0.000 -6.1 [-10.9, -1.8] p=0.010 +2.3 [ -3.4, +8.2] p=0.470 -7.3 [-11.8, -3.3] p=0.001 -8.4 [-13.1, -4.3] p=0.000 -8.6 [-13.4, -4.4] p=0.000 +regression twin (deterministic) + refine - retrieval + refine + +0.0 [ +0.0, +0.0] p=1.000 +1.1 [ +0.0, +3.1] p=0.504 +28.6 [+19.2,+38.1] p=0.000 -0.0 [ -7.2, +7.2] p=1.000 -2.8 [ -9.4, +4.2] p=0.479 -0.0 [ -3.9, +3.9] p=1.000 -1.9 [ -5.3, +1.1] p=0.381 -1.9 [ -5.6, +1.1] p=0.366 -1.9 [ -5.3, +1.1] p=0.371 +regression twin (deterministic) + refine - generated (U-Net, DDPM-500) + refine + +0.0 [ +0.0, +0.0] p=1.000 +1.1 [ +0.0, +3.1] p=0.503 +76.9 [+70.3,+83.6] p=0.000 +83.3 [+77.5,+88.9] p=0.000 -0.3 [ -4.3, +3.7] p=0.932 +5.9 [ +0.9,+11.4] p=0.031 -1.8 [ -4.5, +0.4] p=0.254 -2.9 [ -5.8, -0.6] p=0.066 -2.9 [ -5.6, -0.6] p=0.062 diff --git a/results/furuta_v4/budget_reg_det_s0_n200.txt b/results/furuta_v4/budget_reg_det_s0_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..8ee36f292e592213ec9cb8b7ce808af19299fe84 --- /dev/null +++ b/results/furuta_v4/budget_reg_det_s0_n200.txt @@ -0,0 +1,8 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 7s plain ok 0.833; 10 failed-certified in 10 plants + [gen_s0] refine done 29s + [gen_s0] refined ok at (5, 10, 20, 40): [0.867, 0.917, 0.933, 0.95] (K=1) (33s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 6s plain ok 0.900; 6 failed-certified in 6 plants + [gen_s0] refine done 24s + [gen_s0] refined ok at (5, 10, 20, 40): [0.917, 0.933, 0.95, 0.967] (K=1) (27s) diff --git a/results/furuta_v4/budget_reg_det_s1_n200.txt b/results/furuta_v4/budget_reg_det_s1_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b0fcc340fde70f71648edeb2d79fbd6e292af17 --- /dev/null +++ b/results/furuta_v4/budget_reg_det_s1_n200.txt @@ -0,0 +1,8 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 7s plain ok 0.850; 9 failed-certified in 9 plants + [gen_s0] refine done 28s + [gen_s0] refined ok at (5, 10, 20, 40): [0.9, 0.917, 0.933, 0.933] (K=1) (31s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 6s plain ok 0.867; 8 failed-certified in 8 plants + [gen_s0] refine done 26s + [gen_s0] refined ok at (5, 10, 20, 40): [0.917, 0.95, 0.967, 0.983] (K=1) (29s) diff --git a/results/furuta_v4/budget_reg_det_s2_n200.txt b/results/furuta_v4/budget_reg_det_s2_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..f96cc4ba2cea2ccc5fd4a520f5ea272442f8db12 --- /dev/null +++ b/results/furuta_v4/budget_reg_det_s2_n200.txt @@ -0,0 +1,8 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 6s plain ok 0.817; 11 failed-certified in 11 plants + [gen_s0] refine done 30s + [gen_s0] refined ok at (5, 10, 20, 40): [0.833, 0.867, 0.917, 0.933] (K=1) (33s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 8s plain ok 0.867; 8 failed-certified in 8 plants + [gen_s0] refine done 27s + [gen_s0] refined ok at (5, 10, 20, 40): [0.9, 0.933, 0.933, 0.967] (K=1) (31s) diff --git a/results/furuta_v4/budget_retrN_n100.txt b/results/furuta_v4/budget_retrN_n100.txt new file mode 100644 index 0000000000000000000000000000000000000000..65d962f57a2c4c18ed0c5ad84f287a005af6c6ca --- /dev/null +++ b/results/furuta_v4/budget_retrN_n100.txt @@ -0,0 +1,8 @@ +seed4242 retr n_sub=100 + [retr] SDP+verify 44s plain ok 0.396; 145 failed-certified in 57 plants + [retr] refine done 79s + [retr] refined ok at (5, 10, 20, 40): [0.75, 0.867, 0.883, 0.917] (K=1) (107s) +seed9001 retr n_sub=100 + [retr] SDP+verify 38s plain ok 0.433; 136 failed-certified in 56 plants + [retr] refine done 57s + [retr] refined ok at (5, 10, 20, 40): [0.833, 0.9, 0.917, 0.933] (K=1) (84s) diff --git a/results/furuta_v4/budget_retrN_n200.txt b/results/furuta_v4/budget_retrN_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..15c864dd94ff2bf7f3864018205935eef0600fc0 --- /dev/null +++ b/results/furuta_v4/budget_retrN_n200.txt @@ -0,0 +1,8 @@ +seed4242 retr n_sub=200 + [retr] SDP+verify 65s plain ok 0.322; 145 failed-certified in 57 plants + [retr] refine done 87s + [retr] refined ok at (5, 10, 20, 40): [0.767, 0.867, 0.9, 0.917] (K=1) (99s) +seed9001 retr n_sub=200 + [retr] SDP+verify 56s plain ok 0.319; 136 failed-certified in 56 plants + [retr] refine done 78s + [retr] refined ok at (5, 10, 20, 40): [0.817, 0.917, 0.917, 0.967] (K=1) (89s) diff --git a/results/furuta_v4/budget_retrN_n400.txt b/results/furuta_v4/budget_retrN_n400.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e635065486edc650c060b26af9532c9b0e6808c --- /dev/null +++ b/results/furuta_v4/budget_retrN_n400.txt @@ -0,0 +1,8 @@ +seed4242 retr n_sub=400 + [retr] SDP+verify 44s plain ok 0.396; 145 failed-certified in 57 plants + [retr] refine done 102s + [retr] refined ok at (5, 10, 20, 40): [0.767, 0.867, 0.9, 0.933] (K=1) (130s) +seed9001 retr n_sub=400 + [retr] SDP+verify 38s plain ok 0.433; 136 failed-certified in 56 plants + [retr] refine done 78s + [retr] refined ok at (5, 10, 20, 40): [0.817, 0.917, 0.917, 0.967] (K=1) (104s) diff --git a/results/furuta_v4/budget_timing_cpu.txt b/results/furuta_v4/budget_timing_cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..714646ed572bdeea5b4595afae265641081754d1 --- /dev/null +++ b/results/furuta_v4/budget_timing_cpu.txt @@ -0,0 +1,32 @@ +# virgo-asl load (16.4560546875, 16.083984375, 16.00244140625) device cpu +sample1 1.5558 +sample4 3.5107 +sample8 5.9910 +sample16 10.1967 +sample1_ddim5 0.0167 +sample4_ddim5 0.0355 +sample8_ddim5 0.0586 +sample16_ddim5 0.1014 +sample1_ddim10 0.0315 +sample4_ddim10 0.0688 +sample8_ddim10 0.1178 +sample16_ddim10 0.2043 +sample1_ddim20 0.0640 +sample4_ddim20 0.1433 +sample8_ddim20 0.2364 +sample16_ddim20 0.4055 +sample1_ddim50 0.1808 +sample4_ddim50 0.3675 +sample8_ddim50 0.6048 +sample16_ddim50 1.0319 +regfwd1 0.00319 +regfwd8 0.01162 +refine1_n100 0.366 (first call incl. compile 8.3) +refine2_n100 0.370 (first call incl. compile 0.7) +refine4_n100 0.359 (first call incl. compile 0.7) +refine1_n200 0.662 (first call incl. compile 1.0) +refine2_n200 0.689 (first call incl. compile 1.0) +refine4_n200 0.698 (first call incl. compile 1.0) +refine1_n400 1.389 (first call incl. compile 1.7) +refine2_n400 1.397 (first call incl. compile 1.8) +refine4_n400 1.454 (first call incl. compile 1.8) diff --git a/results/furuta_v4/budget_timing_cuda.txt b/results/furuta_v4/budget_timing_cuda.txt new file mode 100644 index 0000000000000000000000000000000000000000..94a21b8e933df330d584e7a17312ed5f12ed95c0 --- /dev/null +++ b/results/furuta_v4/budget_timing_cuda.txt @@ -0,0 +1,32 @@ +# virgo-asl load (10.923828125, 7.130859375, 6.99365234375) device cuda +sample1 0.9937 +sample4 0.9415 +sample8 1.1329 +sample16 1.5277 +sample1_ddim5 0.0664 +sample4_ddim5 0.0675 +sample8_ddim5 0.0691 +sample16_ddim5 0.0718 +sample1_ddim10 0.0945 +sample4_ddim10 0.0808 +sample8_ddim10 0.0921 +sample16_ddim10 0.0287 +sample1_ddim20 0.0650 +sample4_ddim20 0.0989 +sample8_ddim20 0.1608 +sample16_ddim20 0.1395 +sample1_ddim50 0.6225 +sample4_ddim50 0.3997 +sample8_ddim50 0.4396 +sample16_ddim50 0.4600 +regfwd1 0.00208 +regfwd8 0.00349 +refine1_n100 1.168 (first call incl. compile 7.9) +refine2_n100 1.143 (first call incl. compile 6.9) +refine4_n100 1.148 (first call incl. compile 6.9) +refine1_n200 2.030 (first call incl. compile 6.0) +refine2_n200 1.935 (first call incl. compile 7.7) +refine4_n200 1.986 (first call incl. compile 7.4) +refine1_n400 3.623 (first call incl. compile 8.0) +refine2_n400 3.598 (first call incl. compile 9.6) +refine4_n400 3.850 (first call incl. compile 9.4) diff --git a/results/furuta_v4/budget_unetA_ema999_ddim5_n200.txt b/results/furuta_v4/budget_unetA_ema999_ddim5_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..49accf85e1a5bcad717894c2dae4896f5f5dc8a4 --- /dev/null +++ b/results/furuta_v4/budget_unetA_ema999_ddim5_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 63s plain ok 0.878; 29 failed-certified in 10 plants + [gen_s0] refine done 85s + [gen_s0] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.95, 0.967] (K=1) (89s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 67s plain ok 0.875; 31 failed-certified in 10 plants + [gen_s1] refine done 88s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.983] (K=1) (93s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 61s plain ok 0.878; 33 failed-certified in 10 plants + [gen_s2] refine done 82s + [gen_s2] refined ok at (5, 10, 20, 40): [0.95, 0.95, 0.967, 0.967] (K=1) (87s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 59s plain ok 0.933; 17 failed-certified in 5 plants + [gen_s0] refine done 75s + [gen_s0] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.95, 0.983] (K=1) (78s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 58s plain ok 0.935; 17 failed-certified in 5 plants + [gen_s1] refine done 74s + [gen_s1] refined ok at (5, 10, 20, 40): [0.95, 0.95, 0.933, 0.983] (K=1) (77s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 57s plain ok 0.934; 16 failed-certified in 4 plants + [gen_s2] refine done 72s + [gen_s2] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.95, 0.983] (K=1) (76s) diff --git a/results/furuta_v4/budget_unetA_ema999_n200.txt b/results/furuta_v4/budget_unetA_ema999_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..270ef0e7fea665bd37c6e1c5533290f1e9a7be66 --- /dev/null +++ b/results/furuta_v4/budget_unetA_ema999_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 71s plain ok 0.907; 23 failed-certified in 8 plants + [gen_s0] refine done 91s + [gen_s0] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.967] (K=1) (95s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 72s plain ok 0.902; 23 failed-certified in 7 plants + [gen_s1] refine done 90s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.967] (K=1) (94s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 72s plain ok 0.910; 24 failed-certified in 8 plants + [gen_s2] refine done 93s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.967] (K=1) (97s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 63s plain ok 0.934; 15 failed-certified in 4 plants + [gen_s0] refine done 78s + [gen_s0] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.95, 0.983] (K=1) (81s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 64s plain ok 0.933; 16 failed-certified in 4 plants + [gen_s1] refine done 79s + [gen_s1] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.95, 1.0] (K=1) (83s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 67s plain ok 0.933; 16 failed-certified in 4 plants + [gen_s2] refine done 82s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.967, 0.95, 1.0] (K=1) (86s) diff --git a/results/furuta_v4/budget_unetAseed1_ema999_ddim10_n200.txt b/results/furuta_v4/budget_unetAseed1_ema999_ddim10_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..10446d5dc5e5a65194a64c592e1fdaa76fb1d0f7 --- /dev/null +++ b/results/furuta_v4/budget_unetAseed1_ema999_ddim10_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 66s plain ok 0.895; 25 failed-certified in 8 plants + [gen_s0] refine done 86s + [gen_s0] refined ok at (5, 10, 20, 40): [0.917, 0.917, 0.967, 0.967] (K=1) (90s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 61s plain ok 0.902; 24 failed-certified in 7 plants + [gen_s1] refine done 80s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.95, 0.967] (K=1) (84s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 66s plain ok 0.909; 22 failed-certified in 7 plants + [gen_s2] refine done 85s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.95] (K=1) (89s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 55s plain ok 0.940; 16 failed-certified in 5 plants + [gen_s0] refine done 71s + [gen_s0] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 0.983] (K=1) (75s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 56s plain ok 0.934; 15 failed-certified in 4 plants + [gen_s1] refine done 71s + [gen_s1] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 0.983] (K=1) (74s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 55s plain ok 0.932; 16 failed-certified in 5 plants + [gen_s2] refine done 71s + [gen_s2] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 0.983] (K=1) (75s) diff --git a/results/furuta_v4/budget_unetAseed1_ema999_ddim5_n200.txt b/results/furuta_v4/budget_unetAseed1_ema999_ddim5_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d311475d146830293b2468e4e20e50493594132 --- /dev/null +++ b/results/furuta_v4/budget_unetAseed1_ema999_ddim5_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 62s plain ok 0.872; 28 failed-certified in 10 plants + [gen_s0] refine done 84s + [gen_s0] refined ok at (5, 10, 20, 40): [0.917, 0.95, 0.95, 0.95] (K=1) (88s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 61s plain ok 0.878; 28 failed-certified in 9 plants + [gen_s1] refine done 82s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.933, 0.967] (K=1) (86s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 63s plain ok 0.884; 29 failed-certified in 9 plants + [gen_s2] refine done 84s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.95, 0.95] (K=1) (89s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 55s plain ok 0.933; 17 failed-certified in 5 plants + [gen_s0] refine done 71s + [gen_s0] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 0.983] (K=1) (75s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 58s plain ok 0.932; 17 failed-certified in 5 plants + [gen_s1] refine done 74s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.95, 0.967, 0.983] (K=1) (78s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 56s plain ok 0.929; 17 failed-certified in 5 plants + [gen_s2] refine done 72s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.967, 0.967, 0.967] (K=1) (76s) diff --git a/results/furuta_v4/budget_unetAseed2_ema999_ddim10_n200.txt b/results/furuta_v4/budget_unetAseed2_ema999_ddim10_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee9d6318eb7a577b5987284721d29de56c969a73 --- /dev/null +++ b/results/furuta_v4/budget_unetAseed2_ema999_ddim10_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 60s plain ok 0.898; 23 failed-certified in 8 plants + [gen_s0] refine done 79s + [gen_s0] refined ok at (5, 10, 20, 40): [0.933, 0.95, 0.967, 0.967] (K=1) (84s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 60s plain ok 0.887; 26 failed-certified in 9 plants + [gen_s1] refine done 81s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.967] (K=1) (85s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 61s plain ok 0.893; 27 failed-certified in 9 plants + [gen_s2] refine done 82s + [gen_s2] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.983, 0.983] (K=1) (86s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 55s plain ok 0.935; 16 failed-certified in 4 plants + [gen_s0] refine done 70s + [gen_s0] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.95, 0.983] (K=1) (74s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 56s plain ok 0.939; 16 failed-certified in 4 plants + [gen_s1] refine done 71s + [gen_s1] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.983, 0.983] (K=1) (75s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 54s plain ok 0.934; 16 failed-certified in 5 plants + [gen_s2] refine done 70s + [gen_s2] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.967, 0.983] (K=1) (73s) diff --git a/results/furuta_v4/budget_unetAseed2_ema999_ddim5_n200.txt b/results/furuta_v4/budget_unetAseed2_ema999_ddim5_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..7d43f23041b135847fbd292dbebba83be20ccc11 --- /dev/null +++ b/results/furuta_v4/budget_unetAseed2_ema999_ddim5_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 62s plain ok 0.875; 30 failed-certified in 9 plants + [gen_s0] refine done 83s + [gen_s0] refined ok at (5, 10, 20, 40): [0.9, 0.933, 0.95, 0.983] (K=1) (88s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 61s plain ok 0.878; 28 failed-certified in 10 plants + [gen_s1] refine done 82s + [gen_s1] refined ok at (5, 10, 20, 40): [0.95, 0.95, 0.983, 0.983] (K=1) (87s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 62s plain ok 0.880; 29 failed-certified in 9 plants + [gen_s2] refine done 83s + [gen_s2] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.983, 0.983] (K=1) (88s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 57s plain ok 0.936; 16 failed-certified in 5 plants + [gen_s0] refine done 72s + [gen_s0] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 0.983] (K=1) (76s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 56s plain ok 0.935; 15 failed-certified in 4 plants + [gen_s1] refine done 71s + [gen_s1] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.983, 0.983] (K=1) (75s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 53s plain ok 0.931; 16 failed-certified in 5 plants + [gen_s2] refine done 69s + [gen_s2] refined ok at (5, 10, 20, 40): [0.967, 0.967, 0.967, 0.983] (K=1) (73s) diff --git a/results/furuta_v4/budget_unetAseed2_ema999_n200.txt b/results/furuta_v4/budget_unetAseed2_ema999_n200.txt new file mode 100644 index 0000000000000000000000000000000000000000..6c78cb5e0dc0bb8c42a9d9dc7231924c78bc1434 --- /dev/null +++ b/results/furuta_v4/budget_unetAseed2_ema999_n200.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 n_sub=200 + [gen_s0] SDP+verify 103s plain ok 0.909; 21 failed-certified in 6 plants + [gen_s0] refine done 129s + [gen_s0] refined ok at (5, 10, 20, 40): [0.933, 0.95, 0.967, 0.967] (K=1) (135s) +seed4242 gen_s1 n_sub=200 + [gen_s1] SDP+verify 107s plain ok 0.905; 23 failed-certified in 8 plants + [gen_s1] refine done 136s + [gen_s1] refined ok at (5, 10, 20, 40): [0.933, 0.933, 0.967, 0.967] (K=1) (143s) +seed4242 gen_s2 n_sub=200 + [gen_s2] SDP+verify 100s plain ok 0.914; 21 failed-certified in 6 plants + [gen_s2] refine done 124s + [gen_s2] refined ok at (5, 10, 20, 40): [0.933, 0.95, 0.967, 0.967] (K=1) (130s) +seed9001 gen_s0 n_sub=200 + [gen_s0] SDP+verify 90s plain ok 0.935; 16 failed-certified in 4 plants + [gen_s0] refine done 112s + [gen_s0] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.967, 1.0] (K=1) (118s) +seed9001 gen_s1 n_sub=200 + [gen_s1] SDP+verify 88s plain ok 0.935; 16 failed-certified in 4 plants + [gen_s1] refine done 108s + [gen_s1] refined ok at (5, 10, 20, 40): [0.95, 0.95, 0.95, 0.983] (K=1) (115s) +seed9001 gen_s2 n_sub=200 + [gen_s2] SDP+verify 91s plain ok 0.933; 16 failed-certified in 4 plants + [gen_s2] refine done 112s + [gen_s2] refined ok at (5, 10, 20, 40): [0.95, 0.967, 0.95, 0.983] (K=1) (118s) diff --git a/results/furuta_v4/ddim_summary.txt b/results/furuta_v4/ddim_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..93503d4395374dd70df7643798a99ade130275fb --- /dev/null +++ b/results/furuta_v4/ddim_summary.txt @@ -0,0 +1,6 @@ +steps K=1: pooled | diff to DDPM-500 [95% CI] p K=4: pooled | diff to DDPM-500 [95% CI] p K=8: pooled | diff to DDPM-500 [95% CI] p K=16: pooled | diff to DDPM-500 [95% CI] p +DDPM500 92.4 93.2 93.8 94.2 +DDIM5 90.6 | -1.9 [-4.3,+0.3] p=0.15 92.7 | -0.6 [-2.2,+0.9] p=0.56 93.7 | -0.1 [-0.9,+0.7] p=1.00 94.3 | +0.1 [-0.7,+0.9] p=1.00 +DDIM10 91.7 | -0.7 [-2.7,+0.8] p=0.52 93.5 | +0.3 [-0.5,+1.1] p=0.69 93.9 | +0.1 [-0.5,+0.7] p=1.00 94.4 | +0.3 [-0.2,+0.9] p=0.50 +DDIM20 91.9 | -0.6 [-1.9,+0.4] p=0.50 93.4 | +0.2 [-0.2,+0.6] p=0.75 93.9 | +0.1 [+0.0,+0.3] p=1.00 94.5 | +0.4 [+0.0,+1.0] p=0.50 +DDIM50 91.9 | -0.6 [-1.5,+0.2] p=0.34 93.2 | +0.0 [-0.6,+0.6] p=1.00 93.5 | -0.3 [-0.9,+0.2] p=0.75 94.4 | +0.3 [-0.3,+1.1] p=1.00 diff --git a/results/furuta_v4/ddim_unetA_ema999_10.txt b/results/furuta_v4/ddim_unetA_ema999_10.txt new file mode 100644 index 0000000000000000000000000000000000000000..cd9dffb4530c7108f90aad18e17e60153e423c57 --- /dev/null +++ b/results/furuta_v4/ddim_unetA_ema999_10.txt @@ -0,0 +1,2 @@ +unetA_ema999 DDIM 10 +K=1: 91.4% K=4: 93.1% K=8: 93.3% K=16: 94.2% diff --git a/results/furuta_v4/ddim_unetA_ema999_5.txt b/results/furuta_v4/ddim_unetA_ema999_5.txt new file mode 100644 index 0000000000000000000000000000000000000000..096e5aac986cf1fe4e4e9c68ff6638f8fa887e9c --- /dev/null +++ b/results/furuta_v4/ddim_unetA_ema999_5.txt @@ -0,0 +1,2 @@ +unetA_ema999 DDIM 5 +K=1: 90.0% K=4: 92.5% K=8: 93.9% K=16: 94.7% diff --git a/results/furuta_v4/ddim_unetA_ema999_50.txt b/results/furuta_v4/ddim_unetA_ema999_50.txt new file mode 100644 index 0000000000000000000000000000000000000000..a886cb00186da4c9c568cb1a9b13dd46a52f2fa1 --- /dev/null +++ b/results/furuta_v4/ddim_unetA_ema999_50.txt @@ -0,0 +1,2 @@ +unetA_ema999 DDIM 50 +K=1: 91.7% K=4: 93.1% K=8: 93.1% K=16: 94.7% diff --git a/results/furuta_v4/ddim_unetAseed1_ema999_50.txt b/results/furuta_v4/ddim_unetAseed1_ema999_50.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0366aa3539eb41cf91c3ba074078f4be2f93514 --- /dev/null +++ b/results/furuta_v4/ddim_unetAseed1_ema999_50.txt @@ -0,0 +1,2 @@ +unetAseed1_ema999 DDIM 50 +K=1: 92.2% K=4: 93.3% K=8: 93.3% K=16: 93.9% diff --git a/results/furuta_v4/ddim_unetAseed2_ema999_10.txt b/results/furuta_v4/ddim_unetAseed2_ema999_10.txt new file mode 100644 index 0000000000000000000000000000000000000000..769158a7b3dd39f14143f2662048d40443a8509e --- /dev/null +++ b/results/furuta_v4/ddim_unetAseed2_ema999_10.txt @@ -0,0 +1,2 @@ +unetAseed2_ema999 DDIM 10 +K=1: 91.9% K=4: 93.9% K=8: 94.4% K=16: 95.0% diff --git a/results/furuta_v4/ddim_unetAseed2_ema999_20.txt b/results/furuta_v4/ddim_unetAseed2_ema999_20.txt new file mode 100644 index 0000000000000000000000000000000000000000..f1bafac851423f5bb305bbd990fe21bfa242a8e6 --- /dev/null +++ b/results/furuta_v4/ddim_unetAseed2_ema999_20.txt @@ -0,0 +1,2 @@ +unetAseed2_ema999 DDIM 20 +K=1: 91.7% K=4: 93.3% K=8: 94.4% K=16: 95.0% diff --git a/results/furuta_v4/ddim_unetAseed2_ema999_5.txt b/results/furuta_v4/ddim_unetAseed2_ema999_5.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d7f7ecb0c4cbcfe47807d950b580e14b6ece32b --- /dev/null +++ b/results/furuta_v4/ddim_unetAseed2_ema999_5.txt @@ -0,0 +1,2 @@ +unetAseed2_ema999 DDIM 5 +K=1: 91.4% K=4: 93.1% K=8: 93.9% K=16: 94.2% diff --git a/results/furuta_v4/ddim_unetAseed2_ema999_50.txt b/results/furuta_v4/ddim_unetAseed2_ema999_50.txt new file mode 100644 index 0000000000000000000000000000000000000000..5bfeb3e01740e96bfdabb6a8c738c6d774141268 --- /dev/null +++ b/results/furuta_v4/ddim_unetAseed2_ema999_50.txt @@ -0,0 +1,2 @@ +unetAseed2_ema999 DDIM 50 +K=1: 91.7% K=4: 93.3% K=8: 94.2% K=16: 94.7% diff --git a/results/furuta_v4/deploy_sidebyside.txt b/results/furuta_v4/deploy_sidebyside.txt new file mode 100644 index 0000000000000000000000000000000000000000..d49c9477c7d74137ec81fcc3427ac25cf80baff7 --- /dev/null +++ b/results/furuta_v4/deploy_sidebyside.txt @@ -0,0 +1,19 @@ +deployment mismatch, K=8, nominal-conditioned generated+P, mean over 3 sampler seeds, n=120 plants; last columns: McNemar (retr-only - model-only, p) of each v4 model's seed 0 vs retrieval +scenario retr NMPC v3e2400 v4unetA v4unetS vs retr v4unetA vs retr v4unetS +nominal 90.0% 95.8% 91.4% 93.6% 95.8% 1-6 p=0.1250 0-7 p=0.0156 +fric0.6 86.7% 95.8% 91.1% 93.6% 95.8% 1-10 p=0.0117 0-11 p=0.0010 +fric1.6 9.2% 64.2% 10.3% 12.8% 10.8% 2-6 p=0.2891 1-4 p=0.3750 +fric2.5 0.0% 29.2% 0.0% 0.0% 0.0% 0-0 p=1.0000 0-0 p=1.0000 +ml_mp-_Lp- 68.3% 94.2% 70.8% 80.0% 76.1% 0-14 p=0.0001 5-13 p=0.0963 +ml_mp-_Lp+ 81.7% 95.8% 89.2% 93.6% 95.6% 1-16 p=0.0003 1-17 p=0.0001 +ml_mp+_Lp- 48.3% 95.0% 51.4% 65.6% 59.7% 4-25 p=0.0001 9-25 p=0.0090 +ml_mp+_Lp+ 75.8% 95.0% 82.8% 91.9% 91.9% 2-22 p=0.0000 5-23 p=0.0009 +both_f1.6_mp-_Lp- 9.2% 55.0% 11.9% 18.3% 14.7% 0-11 p=0.0010 2-9 p=0.0654 +both_f1.6_mp-_Lp+ 19.2% 75.0% 18.9% 23.6% 22.8% 0-6 p=0.0312 2-6 p=0.2891 +both_f1.6_mp+_Lp- 0.8% 52.5% 0.6% 0.0% 0.3% 1-0 p=1.0000 1-0 p=1.0000 +both_f1.6_mp+_Lp+ 1.7% 67.5% 2.2% 3.3% 2.2% 0-2 p=0.5000 1-0 p=1.0000 +both_f2.5_mp-_Lp- 0.0% 21.7% 0.3% 0.0% 0.0% 0-0 p=1.0000 0-0 p=1.0000 +both_f2.5_mp-_Lp+ 0.0% 33.3% 0.0% 0.8% 0.3% 0-1 p=1.0000 0-0 p=1.0000 +both_f2.5_mp+_Lp- 0.0% 18.3% 0.0% 0.6% 0.0% 0-0 p=1.0000 0-0 p=1.0000 +both_f2.5_mp+_Lp+ 0.0% 33.3% 0.0% 0.0% 0.0% 0-0 p=1.0000 0-0 p=1.0000 +max over scenarios of (model mean - retrieval mean), points: v3e2400: +7.5, v4unetA: +17.2, v4unetS: +16.1 diff --git a/results/furuta_v4/disjointness.txt b/results/furuta_v4/disjointness.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e62720b39575ef4ab195119cad4771e13b7c1d1 --- /dev/null +++ b/results/furuta_v4/disjointness.txt @@ -0,0 +1,8 @@ +min over (eval plant, bank row) of sup-norm distance in (mp/0.024, Lp/0.129, log Dr, log Dp); a shared plant would give 0; typical bank NN spacing for scale: +bank LOO median 0.0695 +draw 4242 bank3000 min 0.0145 median NN 0.0761 +draw 4242 extra3000 min 0.0214 median NN 0.0681 +draw 9001 bank3000 min 0.0276 median NN 0.0681 +draw 9001 extra3000 min 0.0150 median NN 0.0696 +draw 777 bank3000 min 0.0197 median NN 0.0711 +draw 777 extra3000 min 0.0196 median NN 0.0655 diff --git a/results/furuta_v4/gate_mlp192k_ema999.txt b/results/furuta_v4/gate_mlp192k_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fc0438f2c35d7d7065bbc26451ce1e9a9c6ab7b --- /dev/null +++ b/results/furuta_v4/gate_mlp192k_ema999.txt @@ -0,0 +1,2 @@ +mlp192k_ema999 ema999 models/v4/mlp_192k_s192000.pth +K=1: generated+P 66.7% nearest+P 54.2% K=4: generated+P 87.8% nearest+P 82.5% K=8: generated+P 89.7% nearest+P 90.0% K=16: generated+P 92.5% nearest+P 93.3% diff --git a/results/furuta_v4/gate_mlp384k_ema9995.txt b/results/furuta_v4/gate_mlp384k_ema9995.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffaeb4bf7ff247cb7482de5e1b11994c53312657 --- /dev/null +++ b/results/furuta_v4/gate_mlp384k_ema9995.txt @@ -0,0 +1,2 @@ +mlp384k_ema9995 ema9995 models/v4/mlp_384k_s384000.pth +K=1: generated+P 73.1% nearest+P 54.2% K=4: generated+P 89.4% nearest+P 82.5% K=8: generated+P 92.8% nearest+P 90.0% K=16: generated+P 95.3% nearest+P 93.3% diff --git a/results/furuta_v4/gate_mlpA_ema999.txt b/results/furuta_v4/gate_mlpA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..37e04141c8ca667ad9635e10e54f0a1f7937d20e --- /dev/null +++ b/results/furuta_v4/gate_mlpA_ema999.txt @@ -0,0 +1,2 @@ +mlpA_ema999 ema999 models/v4/mlp_a_s96000.pth +K=1: generated+P 60.8% nearest+P 54.2% K=4: generated+P 87.8% nearest+P 82.5% K=8: generated+P 91.4% nearest+P 90.0% K=16: generated+P 92.8% nearest+P 93.3% diff --git a/results/furuta_v4/gate_mlpA_ema9995.txt b/results/furuta_v4/gate_mlpA_ema9995.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac5c0bb9dbdc4adf86e4f16ebc0ff379ebe1c812 --- /dev/null +++ b/results/furuta_v4/gate_mlpA_ema9995.txt @@ -0,0 +1,2 @@ +mlpA_ema9995 ema9995 models/v4/mlp_a_s96000.pth +K=1: generated+P 55.3% nearest+P 54.2% K=4: generated+P 87.2% nearest+P 82.5% K=8: generated+P 91.1% nearest+P 90.0% K=16: generated+P 93.1% nearest+P 93.3% diff --git a/results/furuta_v4/gate_mlpA_raw.txt b/results/furuta_v4/gate_mlpA_raw.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a142357e75eb2d5d63694ef334a67df9e6f3535 --- /dev/null +++ b/results/furuta_v4/gate_mlpA_raw.txt @@ -0,0 +1,2 @@ +mlpA_raw raw models/v4/mlp_a_s96000.pth +K=1: generated+P 55.6% nearest+P 54.2% K=4: generated+P 87.2% nearest+P 82.5% K=8: generated+P 91.1% nearest+P 90.0% K=16: generated+P 93.1% nearest+P 93.3% diff --git a/results/furuta_v4/gate_mlpXD_ema999.txt b/results/furuta_v4/gate_mlpXD_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..1ace7bd9556f7b8ca6a0dfe27cd2232931e4af64 --- /dev/null +++ b/results/furuta_v4/gate_mlpXD_ema999.txt @@ -0,0 +1,2 @@ +mlpXD_ema999 ema999 models/v4/mlp_xd_s80000.pth +K=1: generated+P 76.7% nearest+P 67.5% K=4: generated+P 91.4% nearest+P 90.0% K=8: generated+P 93.9% nearest+P 94.2% K=16: generated+P 95.0% nearest+P 94.2% diff --git a/results/furuta_v4/gate_unetA_ema999.txt b/results/furuta_v4/gate_unetA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..46a83a142d802de0774f878b92916c1c494bcedd --- /dev/null +++ b/results/furuta_v4/gate_unetA_ema999.txt @@ -0,0 +1,2 @@ +unetA_ema999 ema999 models/v4/unet_a_s96000.pth +K=1: generated+P 91.9% nearest+P 54.2% K=4: generated+P 93.3% nearest+P 82.5% K=8: generated+P 93.6% nearest+P 90.0% K=16: generated+P 93.9% nearest+P 93.3% diff --git a/results/furuta_v4/gate_unetAseed1_ema999.txt b/results/furuta_v4/gate_unetAseed1_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..cd8e49701325d6c3a012cd0cca1dee9d2bbbbd3b --- /dev/null +++ b/results/furuta_v4/gate_unetAseed1_ema999.txt @@ -0,0 +1,2 @@ +unetAseed1_ema999 ema999 models/v4/unet_a_seed1_s96000.pth +K=1: generated+P 93.3% nearest+P 54.2% K=4: generated+P 93.9% nearest+P 82.5% K=8: generated+P 94.2% nearest+P 90.0% K=16: generated+P 94.2% nearest+P 93.3% diff --git a/results/furuta_v4/gate_unetSXD_ema999.txt b/results/furuta_v4/gate_unetSXD_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..6d539c7099ad0ba6e4e277d82af9afd14b02faba --- /dev/null +++ b/results/furuta_v4/gate_unetSXD_ema999.txt @@ -0,0 +1,2 @@ +unetSXD_ema999 ema999 models/v4/unet_sxd_s40000.pth +K=1: generated+P 90.6% nearest+P 67.5% K=4: generated+P 95.3% nearest+P 90.0% K=8: generated+P 96.4% nearest+P 94.2% K=16: generated+P 96.9% nearest+P 94.2% diff --git a/results/furuta_v4/memo_mlpA_ema999.txt b/results/furuta_v4/memo_mlpA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..821fc87de5369072cbd9feec2cdf04dc2536d44f --- /dev/null +++ b/results/furuta_v4/memo_mlpA_ema999.txt @@ -0,0 +1,6 @@ +bank rows 2801; bank-to-bank NN (leave-one-out) RMS: median 0.0444 p10 0.0201 p90 0.1061 +sample diversity: mean pairwise RMS between the 16 samples of one plant (seed 0): median over plants 0.1111 (bank LOO NN median 0.0444) +generated (candidate 1, 360 = 2 draws x 60 plants x seeds): dist to nearest training row: median 0.0729 p10 0.0485 p90 0.1549 + ratio median(d_gen)/median(d_loo) = 1.64; frac of candidates closer than the 10th pct of bank LOO: 0.000; frac closer than 0.25*median LOO: 0.000 + distance to the same-plant retrieved row: median 0.1259 + succeeded K=1: median d_gen 0.0663 (n=219); failed: 0.0824 (n=141) diff --git a/results/furuta_v4/memo_unetA_ema999.txt b/results/furuta_v4/memo_unetA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac3f3badb844ba6f1def40f7f544063353a84ef8 --- /dev/null +++ b/results/furuta_v4/memo_unetA_ema999.txt @@ -0,0 +1,7 @@ +bank rows 2801; bank-to-bank NN (leave-one-out) RMS: median 0.0444 p10 0.0201 p90 0.1061 +sample diversity: mean pairwise RMS between the 16 samples of one plant (seed 0): median over plants 0.0286 (bank LOO NN median 0.0444) +generated (candidate 1, 360 = 2 draws x 60 plants x seeds): dist to nearest training row: median 0.0503 p10 0.0226 p90 0.0947 + ratio median(d_gen)/median(d_loo) = 1.13; frac of candidates closer than the 10th pct of bank LOO: 0.072; frac closer than 0.25*median LOO: 0.003 + distance to the same-plant retrieved row: median 0.1138 + succeeded K=1: median d_gen 0.0478 (n=331); failed: 0.1118 (n=29) +copy test (threshold 0.0111 = 0.25 x median bank LOO): successes with d_gen refine + [gen_s0] refine done 369s + [gen_s0] refined ok 0.983 (377s); median V_min 2.95 -> 0.56 +seed4242 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 58s; plain ok 0.900; 24 failed with a certified P -> refine + [gen_s1] refine done 323s + [gen_s1] refined ok 0.983 (331s); median V_min 3.46 -> 0.62 +seed4242 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 47s; plain ok 0.875; 30 failed with a certified P -> refine + [gen_s2] refine done 340s + [gen_s2] refined ok 0.983 (350s); median V_min 2.62 -> 0.44 +seed4242 retr: ranking 16 candidates + [retr] SDP+verify 78s; plain ok 0.767; 56 failed with a certified P -> refine + [retr] refine done 400s + [retr] refined ok 0.975 (410s); median V_min 1.35 -> 0.55 +seed9001 gen_s0: ranking 16 candidates + [gen_s0] SDP+verify 64s; plain ok 0.942; 14 failed with a certified P -> refine + [gen_s0] refine done 290s + [gen_s0] refined ok 0.971 (295s); median V_min 7.63 -> 2.69 +seed9001 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 43s; plain ok 0.929; 17 failed with a certified P -> refine + [gen_s1] refine done 180s + [gen_s1] refined ok 0.983 (183s); median V_min 6.26 -> 1.17 +seed9001 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 36s; plain ok 0.925; 18 failed with a certified P -> refine + [gen_s2] refine done 199s + [gen_s2] refined ok 0.971 (204s); median V_min 6.22 -> 0.86 +seed9001 retr: ranking 16 candidates + [retr] SDP+verify 42s; plain ok 0.783; 52 failed with a certified P -> refine + [retr] refine done 210s + [retr] refined ok 0.971 (216s); median V_min 1.43 -> 0.60 diff --git a/results/furuta_v4/rank_mlp384k_ema999_refine_summary.txt b/results/furuta_v4/rank_mlp384k_ema999_refine_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..e498144ce3e3806e9fb58cd952a5041f02b1e59f --- /dev/null +++ b/results/furuta_v4/rank_mlp384k_ema999_refine_summary.txt @@ -0,0 +1,11 @@ +results/furuta_v4/rank_mlp384k_ema999_refine.npz +K=1: generated plain 95.0 -> refined 97.8 | retrieval plain 90.8 -> refined 97.5 + gen-refined minus retr-refined: +0.3 [+0.0,+0.8] p=1.0000 + gen-plain minus retr-plain: +4.2 [+0.8,+8.3] p=0.0372 + retr refined minus plain: +6.7 [+2.5,+11.7] p=0.0077 + gen refined minus plain: +2.8 [+0.6,+5.6] p=0.0320 +K=4: generated plain 95.3 -> refined 98.3 | retrieval plain 93.3 -> refined 99.2 + gen-refined minus retr-refined: -0.8 [-2.5,+0.0] p=1.0000 + gen-plain minus retr-plain: +1.9 [-0.3,+5.0] p=0.2459 + retr refined minus plain: +5.8 [+1.7,+10.0] p=0.0161 + gen refined minus plain: +3.1 [+0.6,+6.1] p=0.0622 diff --git a/results/furuta_v4/rank_mlp384k_ema999_summary.txt b/results/furuta_v4/rank_mlp384k_ema999_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..e9b896def9ce9b05514c89e24f55c27306307afe --- /dev/null +++ b/results/furuta_v4/rank_mlp384k_ema999_summary.txt @@ -0,0 +1,11 @@ +results/furuta_v4/rank_mlp384k_ema999.npz +K=1: generated plain 95.0 -> refined 95.0 | retrieval plain 90.8 -> refined 90.8 + gen-refined minus retr-refined: +4.2 [+0.6,+8.3] p=0.0396 + gen-plain minus retr-plain: +4.2 [+0.8,+8.3] p=0.0372 + retr refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 + gen refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 +K=4: generated plain 95.3 -> refined 95.3 | retrieval plain 93.3 -> refined 93.3 + gen-refined minus retr-refined: +1.9 [-0.3,+4.7] p=0.2530 + gen-plain minus retr-plain: +1.9 [-0.3,+5.0] p=0.2459 + retr refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 + gen refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 diff --git a/results/furuta_v4/rank_unetA_ema999.txt b/results/furuta_v4/rank_unetA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..88508b58b26f5e4d59c7cbe8338df5ed03219142 --- /dev/null +++ b/results/furuta_v4/rank_unetA_ema999.txt @@ -0,0 +1,32 @@ +seed4242 gen_s0: ranking 16 candidates + [gen_s0] SDP+verify 70s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s0] refine done 70s + [gen_s0] refined ok 0.933 (70s) +seed4242 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 77s; plain ok 0.921; 19 failed with a certified P -> refine + [gen_s1] refine done 77s + [gen_s1] refined ok 0.921 (77s) +seed4242 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 71s; plain ok 0.929; 17 failed with a certified P -> refine + [gen_s2] refine done 71s + [gen_s2] refined ok 0.929 (71s) +seed4242 retr: ranking 16 candidates + [retr] SDP+verify 77s; plain ok 0.767; 56 failed with a certified P -> refine + [retr] refine done 77s + [retr] refined ok 0.767 (77s) +seed9001 gen_s0: ranking 16 candidates + [gen_s0] SDP+verify 66s; plain ok 0.938; 15 failed with a certified P -> refine + [gen_s0] refine done 66s + [gen_s0] refined ok 0.938 (66s) +seed9001 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 69s; plain ok 0.938; 15 failed with a certified P -> refine + [gen_s1] refine done 69s + [gen_s1] refined ok 0.938 (69s) +seed9001 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 68s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s2] refine done 68s + [gen_s2] refined ok 0.933 (68s) +seed9001 retr: ranking 16 candidates + [retr] SDP+verify 59s; plain ok 0.783; 52 failed with a certified P -> refine + [retr] refine done 59s + [retr] refined ok 0.783 (59s) diff --git a/results/furuta_v4/rank_unetA_ema999_refine.txt b/results/furuta_v4/rank_unetA_ema999_refine.txt new file mode 100644 index 0000000000000000000000000000000000000000..df65eecfa6ece0328a11137dd82e4dd20fdc12d9 --- /dev/null +++ b/results/furuta_v4/rank_unetA_ema999_refine.txt @@ -0,0 +1,32 @@ +seed4242 gen_s0: ranking 16 candidates + [gen_s0] SDP+verify 47s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s0] refine done 179s + [gen_s0] refined ok 0.983 (182s); median V_min 3.21 -> 0.84 +seed4242 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 39s; plain ok 0.921; 19 failed with a certified P -> refine + [gen_s1] refine done 215s + [gen_s1] refined ok 0.983 (222s); median V_min 2.87 -> 0.81 +seed4242 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 38s; plain ok 0.929; 17 failed with a certified P -> refine + [gen_s2] refine done 129s + [gen_s2] refined ok 0.983 (132s); median V_min 2.98 -> 0.87 +seed4242 retr: ranking 16 candidates + [retr] SDP+verify 44s; plain ok 0.767; 56 failed with a certified P -> refine + [retr] refine done 195s + [retr] refined ok 0.975 (204s); median V_min 1.35 -> 0.55 +seed9001 gen_s0: ranking 16 candidates + [gen_s0] SDP+verify 55s; plain ok 0.938; 15 failed with a certified P -> refine + [gen_s0] refine done 285s + [gen_s0] refined ok 0.992 (292s); median V_min 6.09 -> 1.50 +seed9001 gen_s1: ranking 16 candidates + [gen_s1] SDP+verify 58s; plain ok 0.938; 15 failed with a certified P -> refine + [gen_s1] refine done 296s + [gen_s1] refined ok 1.000 (301s); median V_min 6.12 -> 1.50 +seed9001 gen_s2: ranking 16 candidates + [gen_s2] SDP+verify 59s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s2] refine done 289s + [gen_s2] refined ok 0.992 (296s); median V_min 5.34 -> 1.14 +seed9001 retr: ranking 16 candidates + [retr] SDP+verify 48s; plain ok 0.783; 52 failed with a certified P -> refine + [retr] refine done 290s + [retr] refined ok 0.971 (298s); median V_min 1.43 -> 0.60 diff --git a/results/furuta_v4/rank_unetA_ema999_refine_summary.txt b/results/furuta_v4/rank_unetA_ema999_refine_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..7fe43fb7f85bdcfc5a1090626182463b20ccd6ab --- /dev/null +++ b/results/furuta_v4/rank_unetA_ema999_refine_summary.txt @@ -0,0 +1,11 @@ +results/furuta_v4/rank_unetA_ema999_refine.npz +K=1: generated plain 93.9 -> refined 99.2 | retrieval plain 90.8 -> refined 97.5 + gen-refined minus retr-refined: +1.7 [+0.0,+4.2] p=0.5010 + gen-plain minus retr-plain: +3.1 [-1.1,+7.5] p=0.2878 + retr refined minus plain: +6.7 [+2.5,+11.7] p=0.0077 + gen refined minus plain: +5.3 [+1.7,+9.4] p=0.0155 +K=4: generated plain 93.9 -> refined 99.2 | retrieval plain 93.3 -> refined 99.2 + gen-refined minus retr-refined: +0.0 [+0.0,+0.0] p=1.0000 + gen-plain minus retr-plain: +0.6 [-2.8,+4.2] p=1.0000 + retr refined minus plain: +5.8 [+1.7,+10.0] p=0.0161 + gen refined minus plain: +5.3 [+1.7,+9.4] p=0.0148 diff --git a/results/furuta_v4/rank_unetA_ema999_summary.txt b/results/furuta_v4/rank_unetA_ema999_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..9a64530ed0345928ed80d10cc93f5ea0ac92b339 --- /dev/null +++ b/results/furuta_v4/rank_unetA_ema999_summary.txt @@ -0,0 +1,11 @@ +results/furuta_v4/rank_unetA_ema999.npz +K=1: generated plain 93.9 -> refined 93.9 | retrieval plain 90.8 -> refined 90.8 + gen-refined minus retr-refined: +3.1 [-1.1,+7.8] p=0.2858 + gen-plain minus retr-plain: +3.1 [-1.1,+7.5] p=0.2878 + retr refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 + gen refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 +K=4: generated plain 93.9 -> refined 93.9 | retrieval plain 93.3 -> refined 93.3 + gen-refined minus retr-refined: +0.6 [-2.8,+3.9] p=1.0000 + gen-plain minus retr-plain: +0.6 [-2.8,+4.2] p=1.0000 + retr refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 + gen refined minus plain: +0.0 [+0.0,+0.0] p=1.0000 diff --git a/results/furuta_v4/refine_mlp384k_ema999.txt b/results/furuta_v4/refine_mlp384k_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..8212e3399afb6193cf3feb97263285886a1a8530 --- /dev/null +++ b/results/furuta_v4/refine_mlp384k_ema999.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 + [gen_s0] SDP+verify 55s; plain ok 0.700; 72 failed with a certified P -> refine + [gen_s0] refine done 210s + [gen_s0] refined ok 0.950 (218s); median V_min 4.02 -> 0.78 +seed4242 gen_s1 + [gen_s1] SDP+verify 44s; plain ok 0.700; 72 failed with a certified P -> refine + [gen_s1] refine done 198s + [gen_s1] refined ok 0.963 (207s); median V_min 3.41 -> 0.74 +seed4242 gen_s2 + [gen_s2] SDP+verify 50s; plain ok 0.700; 72 failed with a certified P -> refine + [gen_s2] refine done 198s + [gen_s2] refined ok 0.967 (207s); median V_min 2.76 -> 0.74 +seed9001 gen_s0 + [gen_s0] SDP+verify 57s; plain ok 0.796; 49 failed with a certified P -> refine + [gen_s0] refine done 202s + [gen_s0] refined ok 0.954 (208s); median V_min 3.17 -> 0.77 +seed9001 gen_s1 + [gen_s1] SDP+verify 52s; plain ok 0.779; 53 failed with a certified P -> refine + [gen_s1] refine done 188s + [gen_s1] refined ok 0.971 (195s); median V_min 2.67 -> 0.78 +seed9001 gen_s2 + [gen_s2] SDP+verify 43s; plain ok 0.787; 51 failed with a certified P -> refine + [gen_s2] refine done 182s + [gen_s2] refined ok 0.971 (188s); median V_min 2.62 -> 0.77 diff --git a/results/furuta_v4/refine_mlp384k_ema999_summary.txt b/results/furuta_v4/refine_mlp384k_ema999_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..102d9578cea01191dc5a9177ec062b877b948868 --- /dev/null +++ b/results/furuta_v4/refine_mlp384k_ema999_summary.txt @@ -0,0 +1,12 @@ +results/furuta_v4/refine_mlp384k_ema999.npz +retrieval arm taken from results/furuta_v4/refine_mlpA_ema999.npz +K=1: generated plain 73.3 -> refined 95.8 | retrieval plain 54.2 -> refined 95.0 + gen-refined minus retr-refined: +0.8 [-2.5,+4.7] p=0.7703 + gen-plain minus retr-plain: +19.2 [+9.7,+28.9] p=0.0003 + retr refined minus plain: +40.8 [+32.5,+50.0] p=0.0000 + gen refined minus plain: +22.5 [+16.7,+28.6] p=0.0000 +K=4: generated plain 89.4 -> refined 98.1 | retrieval plain 82.5 -> refined 97.5 + gen-refined minus retr-refined: +0.6 [-0.8,+2.5] p=1.0000 + gen-plain minus retr-plain: +6.9 [+1.7,+12.8] p=0.0187 + retr refined minus plain: +15.0 [+9.2,+21.7] p=0.0000 + gen refined minus plain: +8.6 [+4.7,+13.1] p=0.0001 diff --git a/results/furuta_v4/refine_mlpA_ema999.txt b/results/furuta_v4/refine_mlpA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe2c84af2f59c28f04437317440c422d905ff02f --- /dev/null +++ b/results/furuta_v4/refine_mlpA_ema999.txt @@ -0,0 +1,32 @@ +seed4242 gen_s0 + [gen_s0] SDP+verify 69s; plain ok 0.562; 105 failed with a certified P -> refine + [gen_s0] refine done 321s + [gen_s0] refined ok 0.921 (335s); median V_min 3.53 -> 0.74 +seed4242 gen_s1 + [gen_s1] SDP+verify 64s; plain ok 0.613; 93 failed with a certified P -> refine + [gen_s1] refine done 284s + [gen_s1] refined ok 0.938 (295s); median V_min 2.87 -> 0.75 +seed4242 gen_s2 + [gen_s2] SDP+verify 55s; plain ok 0.583; 100 failed with a certified P -> refine + [gen_s2] refine done 263s + [gen_s2] refined ok 0.938 (273s); median V_min 3.64 -> 0.74 +seed4242 retr + [retr] SDP+verify 55s; plain ok 0.396; 145 failed with a certified P -> refine + [retr] refine done 472s + [retr] refined ok 0.900 (485s); median V_min 2.44 -> 0.71 +seed9001 gen_s0 + [gen_s0] SDP+verify 52s; plain ok 0.558; 106 failed with a certified P -> refine + [gen_s0] refine done 270s + [gen_s0] refined ok 0.938 (282s); median V_min 2.79 -> 0.78 +seed9001 gen_s1 + [gen_s1] SDP+verify 55s; plain ok 0.650; 84 failed with a certified P -> refine + [gen_s1] refine done 279s + [gen_s1] refined ok 0.958 (290s); median V_min 2.85 -> 0.74 +seed9001 gen_s2 + [gen_s2] SDP+verify 48s; plain ok 0.583; 100 failed with a certified P -> refine + [gen_s2] refine done 241s + [gen_s2] refined ok 0.950 (252s); median V_min 2.73 -> 0.77 +seed9001 retr + [retr] SDP+verify 48s; plain ok 0.433; 136 failed with a certified P -> refine + [retr] refine done 447s + [retr] refined ok 0.938 (459s); median V_min 2.49 -> 0.74 diff --git a/results/furuta_v4/refine_unetA_ema999.txt b/results/furuta_v4/refine_unetA_ema999.txt new file mode 100644 index 0000000000000000000000000000000000000000..9ec85e9a97b2931e68beca80ccfce3fab6046311 --- /dev/null +++ b/results/furuta_v4/refine_unetA_ema999.txt @@ -0,0 +1,24 @@ +seed4242 gen_s0 + [gen_s0] SDP+verify 70s; plain ok 0.904; 23 failed with a certified P -> refine + [gen_s0] refine done 264s + [gen_s0] refined ok 0.975 (272s); median V_min 3.26 -> 0.91 +seed4242 gen_s1 + [gen_s1] SDP+verify 73s; plain ok 0.904; 23 failed with a certified P -> refine + [gen_s1] refine done 258s + [gen_s1] refined ok 0.975 (265s); median V_min 3.04 -> 0.77 +seed4242 gen_s2 + [gen_s2] SDP+verify 71s; plain ok 0.900; 24 failed with a certified P -> refine + [gen_s2] refine done 225s + [gen_s2] refined ok 0.975 (231s); median V_min 3.22 -> 0.83 +seed9001 gen_s0 + [gen_s0] SDP+verify 59s; plain ok 0.938; 15 failed with a certified P -> refine + [gen_s0] refine done 218s + [gen_s0] refined ok 0.988 (223s); median V_min 6.14 -> 1.50 +seed9001 gen_s1 + [gen_s1] SDP+verify 53s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s1] refine done 193s + [gen_s1] refined ok 0.996 (198s); median V_min 5.85 -> 1.14 +seed9001 gen_s2 + [gen_s2] SDP+verify 54s; plain ok 0.933; 16 failed with a certified P -> refine + [gen_s2] refine done 212s + [gen_s2] refined ok 0.992 (218s); median V_min 5.58 -> 1.13 diff --git a/results/furuta_v4/refine_unetA_ema999_summary.txt b/results/furuta_v4/refine_unetA_ema999_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..59da9a52b6eff3e460806cb711668dd813ad1ce6 --- /dev/null +++ b/results/furuta_v4/refine_unetA_ema999_summary.txt @@ -0,0 +1,12 @@ +results/furuta_v4/refine_unetA_ema999.npz +retrieval arm taken from results/furuta_v4/refine_mlpA_ema999.npz +K=1: generated plain 91.9 -> refined 98.1 | retrieval plain 54.2 -> refined 95.0 + gen-refined minus retr-refined: +3.1 [-0.6,+7.0] p=0.2216 + gen-plain minus retr-plain: +37.8 [+28.9,+46.9] p=0.0000 + retr refined minus plain: +40.8 [+32.5,+50.0] p=0.0000 + gen refined minus plain: +6.1 [+2.2,+10.3] p=0.0033 +K=4: generated plain 93.3 -> refined 99.2 | retrieval plain 82.5 -> refined 97.5 + gen-refined minus retr-refined: +1.7 [+0.0,+4.2] p=0.5020 + gen-plain minus retr-plain: +10.8 [+5.0,+17.2] p=0.0006 + retr refined minus plain: +15.0 [+9.2,+21.7] p=0.0000 + gen refined minus plain: +5.8 [+2.2,+10.0] p=0.0069 diff --git a/results/furuta_v4/reg_attribution.txt b/results/furuta_v4/reg_attribution.txt new file mode 100644 index 0000000000000000000000000000000000000000..3614dc708c360bc1f28610301fad38494caefe96 --- /dev/null +++ b/results/furuta_v4/reg_attribution.txt @@ -0,0 +1,36 @@ +== pooled success % (120 plants); per-training-seed values after the slash == +method K=1 K=2 K=3 K=4 K=8 +nearest+P retrieval 54.2 82.5 90.0 +diffusion unet_a (9 runs; per train seed) 92.4 / 91.9/93.3/91.9 93.2 / 93.3/93.9/92.5 93.8 / 93.6/94.2/93.6 +regression twin, deterministic (3 seeds) 85.6 / 86.7/85.8/84.2 +regression ens3 (members as K candidates) 85.6 87.5 88.3 +regression MC-dropout p=.1 (9 runs) 86.0 / 86.1/85.8/86.1 88.0 / 87.5/88.3/88.1 88.2 / 87.8/88.6/88.3 + same net, dropout OFF (K=1) 86.9 / 87.5/87.5/85.8 + +== paired tests, per-plant seed-averaged difference A-B (pp), bootstrap 95% CI, sign-flip p == +K=1: diffusion - retrieval +38.2 [+29.2,+46.9] p=0.0000 +K=4: diffusion - retrieval +10.7 [ +5.0,+16.7] p=0.0006 +K=8: diffusion - retrieval +3.8 [ +0.0, +8.0] p=0.0858 +K=1: regression det - retrieval +31.4 [+23.1,+40.0] p=0.0000 +K=1: diffusion - regression det +6.9 [ +3.3,+10.9] p=0.0001 +K=1: diffusion - MC-dropout net +6.4 [ +3.1,+10.2] p=0.0001 +K=1: regression det - MC-dropout net (K=1) -0.5 [ -2.6, +1.6] p=0.7410 +K=4: diffusion - MC-dropout +5.3 [ +1.9, +9.2] p=0.0019 +K=4: MC-dropout - retrieval +5.5 [ +0.4,+10.9] p=0.0426 +K=8: diffusion - MC-dropout +5.6 [ +2.2, +9.7] p=0.0039 +K=8: MC-dropout - retrieval -1.8 [ -6.5, +2.6] p=0.4941 +ens3 K=3 - diffusion K=4 (diffusion gets MORE candidates) -4.9 [ -9.0, -1.6] p=0.0159 +ens3 K=3 - regression det K=1 +2.8 [ +0.8, +5.3] p=0.0154 +MC-dropout K=4 - regression det K=1 +2.4 [ +0.0, +5.2] p=0.0803 +diffusion K=4 - diffusion K=1 +0.8 [ +0.2, +1.8] p=0.0626 + +== exact McNemar, single runs on 120 plants (b = diffusion only, c = regression only) == +train seed 0: diffusion(samp0) 92.5 vs det 86.7 b=7 c=0 p=0.016 +train seed 1: diffusion(samp0) 94.2 vs det 85.8 b=10 c=0 p=0.002 +train seed 2: diffusion(samp0) 92.5 vs det 84.2 b=10 c=0 p=0.002 +majority over runs: diffusion 92.5 vs det 85.8 b=8 c=0 p=0.008 +K=1 majority diffusion vs retrieval b=47 c=1 p=3.5e-13 +K=4 majority diffusion vs retrieval b=14 c=1 p=0.00098 +K=1 majority regression det vs retrieval b=40 c=2 p=4.1e-10 + +diversity (pairwise RMS of 8 MC-dropout samples, normalised space, mean over plants): 0.034 (diffusion unet_a 0.029, bank NN 0.044) diff --git a/results/furuta_v4/run_ood.sh b/results/furuta_v4/run_ood.sh new file mode 100644 index 0000000000000000000000000000000000000000..37fe5ed8ae7d8efc8e1fd97d9d61615763c08522 --- /dev/null +++ b/results/furuta_v4/run_ood.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd /home/cercola/matte_server/DiffusionControl; export PYTHONPATH=. OMP_NUM_THREADS=1 CUDA_VISIBLE_DEVICES=0 FURUTA_GEN_DEVICE=cuda +myenv/bin/python -u -m nonlinear.experiments.furuta_v4_ood --stage eval --workers 5 && myenv/bin/python -m nonlinear.experiments.furuta_v4_ood --stage report diff --git a/results/furuta_v4/run_warm_deploy.sh b/results/furuta_v4/run_warm_deploy.sh new file mode 100644 index 0000000000000000000000000000000000000000..b6b080c2fc9865a3a03ed4948b21af2feb7573b7 --- /dev/null +++ b/results/furuta_v4/run_warm_deploy.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /home/cercola/matte_server/DiffusionControl; export PYTHONPATH=. OMP_NUM_THREADS=1 CUDA_VISIBLE_DEVICES=0 FURUTA_GEN_DEVICE=cuda +for m in "unetA models/v4/unet_a_s96000.pth" "unetS models/v4/unet_s_s40000.pth"; do set -- $m + for s in 0 1 2; do myenv/bin/python -u -m nonlinear.experiments.furuta_v4_warm_deploy warm --ckpt $2 --clip 1 --weights ema --tag v4$1_s$s --sampler-seed $s --workers 6; done + myenv/bin/python -u -m nonlinear.experiments.furuta_v4_warm_deploy deploy --ckpt $2 --clip 1 --weights ema --tag v4$1 --workers 6 --n 60 +done diff --git a/results/furuta_v4/sample_gpu.sh b/results/furuta_v4/sample_gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..c055ad2ffb7c68a043ef22b06a75c33cf9b18e24 --- /dev/null +++ b/results/furuta_v4/sample_gpu.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# GPU-only candidate sampling into the cache (scoring happens later on CPU). usage: sample_gpu.sh plants name ckpt weights ... +cd /home/cercola/matte_server/DiffusionControl; export PYTHONPATH=. OMP_NUM_THREADS=1 CUDA_VISIBLE_DEVICES=0 +pl=$1; shift +while [ $# -gt 0 ]; do + [ "$(date +%H)" -lt 9 ] || { echo "past 09:00 stop"; exit 0; } + myenv/bin/python -u -m nonlinear.experiments.furuta_v4_eval --plants $pl --name $1 --ckpt $2 --weights $3 --device cuda --sample-only --workers 1; shift 3 +done diff --git a/results/furuta_v4/scan_mlp_192k.txt b/results/furuta_v4/scan_mlp_192k.txt new file mode 100644 index 0000000000000000000000000000000000000000..6ae92d0f2023495788aa037931c9e4d75ef46f87 --- /dev/null +++ b/results/furuta_v4/scan_mlp_192k.txt @@ -0,0 +1,10 @@ +mlp_192k_s48000_ema999 K1=40.0 K4=77.5 K8=85.0 nearK1=57.5 nearK4=85.0 score=58.7 +mlp_192k_s48000_ema9995 K1=41.2 K4=73.8 K8=87.5 nearK1=57.5 nearK4=85.0 score=57.5 +mlp_192k_s64000_ema999 K1=38.8 K4=76.2 K8=91.3 nearK1=57.5 nearK4=85.0 score=57.5 +mlp_192k_s64000_ema9995 K1=41.2 K4=78.8 K8=91.3 nearK1=57.5 nearK4=85.0 score=60.0 +mlp_192k_s80000_ema999 K1=41.2 K4=82.5 K8=91.3 nearK1=57.5 nearK4=85.0 score=61.9 +mlp_192k_s80000_ema9995 K1=38.8 K4=80.0 K8=90.0 nearK1=57.5 nearK4=85.0 score=59.4 +mlp_192k_s96000_ema999 K1=42.5 K4=81.2 K8=93.8 nearK1=57.5 nearK4=85.0 score=61.9 +mlp_192k_s96000_ema9995 K1=45.0 K4=83.8 K8=93.8 nearK1=57.5 nearK4=85.0 score=64.4 +mlp_192k_s112000_ema999 K1=47.5 K4=86.2 K8=91.2 nearK1=57.5 nearK4=85.0 score=66.9 +mlp_192k_s112000_ema9995 K1=46.2 K4=86.2 K8=90.0 nearK1=57.5 nearK4=85.0 score=66.2 diff --git a/results/furuta_v4/scan_mlp_384k.txt b/results/furuta_v4/scan_mlp_384k.txt new file mode 100644 index 0000000000000000000000000000000000000000..1063b65a9de859b3a0dfcd887fd8e14ee37f888c --- /dev/null +++ b/results/furuta_v4/scan_mlp_384k.txt @@ -0,0 +1,8 @@ +mlp_384k_s192000_ema999 K1=66.3 K4=91.3 K8=92.5 nearK1=57.5 nearK4=85.0 score=78.8 +mlp_384k_s224000_ema999 K1=70.0 K4=95.0 K8=97.5 nearK1=57.5 nearK4=85.0 score=82.5 +mlp_384k_s256000_ema999 K1=71.2 K4=93.8 K8=97.5 nearK1=57.5 nearK4=85.0 score=82.5 +mlp_384k_s288000_ema999 K1=73.8 K4=96.2 K8=98.8 nearK1=57.5 nearK4=85.0 score=85.0 +mlp_384k_s320000_ema999 K1=75.0 K4=92.5 K8=98.8 nearK1=57.5 nearK4=85.0 score=83.8 +mlp_384k_s352000_ema999 K1=76.2 K4=91.3 K8=98.8 nearK1=57.5 nearK4=85.0 score=83.8 +mlp_384k_s384000_ema999 K1=77.5 K4=93.8 K8=98.8 nearK1=57.5 nearK4=85.0 score=85.6 +BEST mlp_384k_s384000_ema999 K1=77.5 K4=93.8 K8=98.8 nearK1=57.5 nearK4=85.0 score=85.6 diff --git a/results/furuta_v4/timing.txt b/results/furuta_v4/timing.txt new file mode 100644 index 0000000000000000000000000000000000000000..86a977a2f7ebed337f2c0ccbfaa3c0825de85bf5 --- /dev/null +++ b/results/furuta_v4/timing.txt @@ -0,0 +1,9 @@ +mlp384k sample1: 5.97 s CPU per plant +mlp384k sample16: 6.72 s CPU per plant +unet_s sample1: 1.63 s CPU per plant +unet_s sample16: 3.93 s CPU per plant +unet_a sample1: 2.17 s CPU per plant +unet_a sample16: 11.38 s CPU per plant +sdp+verify one candidate: 0.42 s +rank16 (open-loop sims of 16 candidates): 0.48 s +refine40 one candidate: 211.27 s diff --git a/results/furuta_v4/traj_gen_warmstart_v4unetA_s2.txt b/results/furuta_v4/traj_gen_warmstart_v4unetA_s2.txt new file mode 100644 index 0000000000000000000000000000000000000000..54e8a1e144361ea1560ef908cb1d0212a7c2ec46 --- /dev/null +++ b/results/furuta_v4/traj_gen_warmstart_v4unetA_s2.txt @@ -0,0 +1,17 @@ +=== warm-start test (n=60, seed 4242, max_iter=700) === +target certificate: largest_certified_region per plant (unmargined); verifier: furuta_traj_bank.score_solution (identical to every other arm in this doc) + + condition success med iter med t med J + cold 53.3% 363 1.93s 4.06 + nearest-warm 76.7% 45 0.25s 4.15 + generated-warm 76.7% 49 0.27s 4.15 + +--- paired tests --- + generated-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.167x (n=32), Wilcoxon p=0.0000 * + generated-warm vs nearest-warm success: beats 0, loses 0, McNemar p=1.0000 time ratio 1.076x (n=46), Wilcoxon p=0.3916 + nearest-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.162x (n=32), Wilcoxon p=0.0000 * + +--- failure breakdown --- + cold ipopt: Maximum_Iterations_Exceeded:13 ipopt: Infeasible_Problem_Detected:11 rollout failed:4 + nearest-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:3 + generated-warm ipopt: Infeasible_Problem_Detected:7 ipopt: Maximum_Iterations_Exceeded:4 rollout failed:3 diff --git a/results/furuta_v4/traj_gen_warmstart_v4unetS_s0.txt b/results/furuta_v4/traj_gen_warmstart_v4unetS_s0.txt new file mode 100644 index 0000000000000000000000000000000000000000..38e0c9cae095f3e4d214e2a0891b681d344ee81f --- /dev/null +++ b/results/furuta_v4/traj_gen_warmstart_v4unetS_s0.txt @@ -0,0 +1,17 @@ +=== warm-start test (n=60, seed 4242, max_iter=700) === +target certificate: largest_certified_region per plant (unmargined); verifier: furuta_traj_bank.score_solution (identical to every other arm in this doc) + + condition success med iter med t med J + cold 53.3% 363 1.83s 4.06 + nearest-warm 76.7% 45 0.26s 4.15 + generated-warm 75.0% 47 0.26s 4.14 + +--- paired tests --- + generated-warm vs cold success: beats 13, loses 0, McNemar p=0.0002 * time ratio 0.162x (n=32), Wilcoxon p=0.0000 * + generated-warm vs nearest-warm success: beats 0, loses 1, McNemar p=1.0000 time ratio 0.994x (n=45), Wilcoxon p=0.5991 + nearest-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.157x (n=32), Wilcoxon p=0.0000 * + +--- failure breakdown --- + cold ipopt: Maximum_Iterations_Exceeded:13 ipopt: Infeasible_Problem_Detected:11 rollout failed:4 + nearest-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:3 + generated-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:4 diff --git a/results/furuta_v4/traj_gen_warmstart_v4unetS_s1.txt b/results/furuta_v4/traj_gen_warmstart_v4unetS_s1.txt new file mode 100644 index 0000000000000000000000000000000000000000..62e59119dbe5e2a4d5376a4dc11acb250c2aa0df --- /dev/null +++ b/results/furuta_v4/traj_gen_warmstart_v4unetS_s1.txt @@ -0,0 +1,17 @@ +=== warm-start test (n=60, seed 4242, max_iter=700) === +target certificate: largest_certified_region per plant (unmargined); verifier: furuta_traj_bank.score_solution (identical to every other arm in this doc) + + condition success med iter med t med J + cold 53.3% 363 1.82s 4.06 + nearest-warm 76.7% 45 0.25s 4.15 + generated-warm 75.0% 50 0.27s 4.14 + +--- paired tests --- + generated-warm vs cold success: beats 13, loses 0, McNemar p=0.0002 * time ratio 0.166x (n=32), Wilcoxon p=0.0000 * + generated-warm vs nearest-warm success: beats 0, loses 1, McNemar p=1.0000 time ratio 1.050x (n=45), Wilcoxon p=0.2618 + nearest-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.154x (n=32), Wilcoxon p=0.0000 * + +--- failure breakdown --- + cold ipopt: Maximum_Iterations_Exceeded:13 ipopt: Infeasible_Problem_Detected:11 rollout failed:4 + nearest-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:3 + generated-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:4 diff --git a/results/furuta_v4/traj_gen_warmstart_v4unetS_s2.txt b/results/furuta_v4/traj_gen_warmstart_v4unetS_s2.txt new file mode 100644 index 0000000000000000000000000000000000000000..cfec7cdc4a6cef4b6c15c26582fb52f88ca881fc --- /dev/null +++ b/results/furuta_v4/traj_gen_warmstart_v4unetS_s2.txt @@ -0,0 +1,17 @@ +=== warm-start test (n=60, seed 4242, max_iter=700) === +target certificate: largest_certified_region per plant (unmargined); verifier: furuta_traj_bank.score_solution (identical to every other arm in this doc) + + condition success med iter med t med J + cold 53.3% 363 1.82s 4.06 + nearest-warm 76.7% 45 0.24s 4.15 + generated-warm 76.7% 48 0.26s 4.15 + +--- paired tests --- + generated-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.151x (n=32), Wilcoxon p=0.0000 * + generated-warm vs nearest-warm success: beats 1, loses 1, McNemar p=1.0000 time ratio 1.064x (n=45), Wilcoxon p=0.7122 + nearest-warm vs cold success: beats 14, loses 0, McNemar p=0.0001 * time ratio 0.164x (n=32), Wilcoxon p=0.0000 * + +--- failure breakdown --- + cold ipopt: Maximum_Iterations_Exceeded:13 ipopt: Infeasible_Problem_Detected:11 rollout failed:4 + nearest-warm ipopt: Infeasible_Problem_Detected:6 rollout failed:5 ipopt: Maximum_Iterations_Exceeded:3 + generated-warm ipopt: Infeasible_Problem_Detected:5 ipopt: Maximum_Iterations_Exceeded:5 rollout failed:4 diff --git a/results/furuta_v4/vs_bigger_bank_retrieval.txt b/results/furuta_v4/vs_bigger_bank_retrieval.txt new file mode 100644 index 0000000000000000000000000000000000000000..6f42ce76e5e259ee804f06209454c1c5a915278b --- /dev/null +++ b/results/furuta_v4/vs_bigger_bank_retrieval.txt @@ -0,0 +1,9 @@ +generator trained on the 2801-plant bank vs nearest+P over the 5583-plant bank (stronger retrieval) +mlp384k_ema999 K=1: gen 73.3 retr(5583) 67.5 diff +5.8 [-3.6,+15.8] p=0.2610 +mlp384k_ema999 K=4: gen 89.4 retr(5583) 90.0 diff -0.6 [-4.7,+3.9] p=0.8989 +mlpB256_ema999 K=1: gen 74.7 retr(5583) 67.5 diff +7.2 [-2.5,+16.7] p=0.1542 +mlpB256_ema999 K=4: gen 90.0 retr(5583) 90.0 diff -0.0 [-3.6,+4.2] p=1.0000 +unetA_ema999 K=1: gen 91.9 retr(5583) 67.5 diff +24.4 [+16.4,+32.8] p=0.0000 +unetA_ema999 K=4: gen 93.3 retr(5583) 90.0 diff +3.3 [-0.8,+8.3] p=0.2112 +unetS_ema999 K=1: gen 90.3 retr(5583) 67.5 diff +22.8 [+14.7,+31.1] p=0.0000 +unetS_ema999 K=4: gen 95.0 retr(5583) 90.0 diff +5.0 [+0.8,+10.0] p=0.0678 diff --git a/results/icml_r2/RULES.md b/results/icml_r2/RULES.md new file mode 100644 index 0000000000000000000000000000000000000000..082453e7ef52e2ef81c0142ae55951b7b5a04b03 --- /dev/null +++ b/results/icml_r2/RULES.md @@ -0,0 +1,42 @@ +# Shared rules for every agent in the ICML round-2 runs (read fully before doing anything) + +Repo: /home/cercola/matte_server/DiffusionControl (branch icml-experiments; do NOT commit or push). +Paper goal: ICML 2027. Diffusion models GENERATE certified controllers per plant (SISO/MIMO Lur'e, pendulum, +Furuta swing-up+catch). Your job is to make the method as strong as possible: baselines (retrieval, regression +twin, NMPC, LQR) are TARGETS to beat, not verdicts. If the model is behind, diagnose (data coverage, scaling of +targets, capacity, EMA, sampler, K, conditioning) and try fixes before concluding. Still report numbers honestly. + +## Reference material (read what you need) +- docs/furuta_generated_trajectory_v4.md (headline recipe + eval protocol; §5 regression twin, §6 budget, §7 fast verify) +- docs/furuta_generated_trajectory_v3.md (clip bug: diffusers DDPMScheduler clip_sample=True,range 1 -> min-max scale targets into [-0.96,0.96]) +- docs/ood_shift.md, docs/statistics_audit.md, docs/timing_vs_classical.md, docs/retrieval_baselines.md, docs/bank_density_siso_mimo.md +- Code: nonlinear/training/train_furuta_v4.py (U-Net/MLP + EMA), train_furuta_v4_reg.py (regression twin), + nonlinear/experiments/furuta_v4_*.py, furuta_fastscore.py (numba scorer), nonlinear/data_generation/furuta_nmpc_bank_v2.py, + nonlinear/data_generation/furuta_roa.py (certified_catch_set, verify_certificate), utils/samplers.py, utils/ddim.py. +- Headline: Furuta U-Net K=1 92-93% vs nearest+P 54%; K=4 92.5-94% vs 82.5%; models/v4/unet_a*_s96000.pth; + bank data/furuta_nmpc_bank_v4merged.npz (5583 plants, params = (mp,Lp,Dr,Dp)); dataset data/furuta_traj_dataset_v4.pt. + +## Environment +- Local python: /home/cercola/matte_server/DiffusionControl/myenv/bin/python (check it imports torch/diffusers/cvxpy/casadi; + fallback /home/cercola/miniconda3/envs/int-act-conda/bin/python with `import numpy as np; np.trapezoid=np.trapz` shim). + Run from repo root with PYTHONPATH=. and OMP_NUM_THREADS=1 for CPU pools. +- andromeda (ssh andromeda, RTX 5090 32GB, 32 threads): repo ~/DiffusionControl, venv ~/DiffusionControl/.venv. + Sync code with: rsync -az --exclude=/data --exclude=/models --exclude=/results --exclude=/figures --exclude=/myenv --exclude=/wandb -e ssh ./ andromeda:~/DiffusionControl/ + (ANCHORED excludes only). Sync specific data/models explicitly. Never put any password in a command. + +## Hard constraints +1. GPU only between 00:00 and 09:00 PT (any day). At 09:00 PT your GPU processes must be finished or stopped + (save checkpoints, resume next window or continue on CPU). Today ends 09:00 PT 2026-10-03. +2. GPUs are SHARED. Before any GPU launch check `nvidia-smi --query-gpu=memory.free --format=csv` and run only on headroom. + NEVER kill/stop/pause any process you did not start. Use `pgrep -f ` not pkill patterns that could match shells. +3. Every job > 2 min must survive session death: `setsid nohup > results/icml_r2//.log 2>&1 < /dev/null & disown`, + write checkpoints/partial results, make it resumable. Then poll with short sleeps (sleep <= 300 in a loop is fine). +4. CPU budget (cores): stated in your task. Respect it (worker counts, OMP_NUM_THREADS=1). +5. Do not change defaults/behaviour of existing shared code; add new files or opt-in arguments only. +6. Outputs: code under nonlinear/... (or the module dir stated), results/logs under results/icml_r2//, + figures under figures/icml_r2/ with NO in-axes explanatory text/boxes/arrows (explanations go in captions). +7. Statistics: held-out plants never in any bank; paired tests (McNemar / sign-flip permutation), bootstrap CIs over plants, + matched budget (same K, same verifier calls) for every arm. Fix seeds and record them. +8. Final deliverable: a doc docs/.md written with the research-doc skill (~/.claude/skills/research-doc/SKILL.md: + Problem/Goal, Method, Results, Conclusion, front-matter). Then return a SHORT report (<=25 lines): headline numbers, + paths, what is still running (with pid/log path) and what is blocked. Return early only if truly blocked. diff --git a/results/inverse_design/log_o2_k4_s0.txt b/results/inverse_design/log_o2_k4_s0.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c53d9e7f9050dd04d478ef5cfdc3bb6157537bb --- /dev/null +++ b/results/inverse_design/log_o2_k4_s0.txt @@ -0,0 +1,22 @@ +bank rows 29199 plants 1397 rows/plant 20.901216893342877 corner frac 0.22120620569197574 +ep0 loss 1.0768 3s +ep10 loss 0.1406 32s +ep20 loss 0.1176 61s +ep30 loss 0.1119 90s +ep40 loss 0.1040 119s +ep50 loss 0.1032 148s +train s 174.47957634925842 +n requests 60 +sampling s 42.86711859703064 +eval s 118.20729184150696 +B15_diffusion hit5%=0.22 hit10%=0.48 med_err=0.111 inval=0.07 +B15_retr_joint hit5%=0.37 hit10%=0.65 med_err=0.064 inval=0.07 +B15_retr_plant_first hit5%=0.17 hit10%=0.27 med_err=0.208 inval=0.17 +B15_random_prior hit5%=0.30 hit10%=0.53 med_err=0.078 inval=0.00 +B100_diffusion hit5%=0.58 hit10%=0.72 med_err=0.037 inval=0.05 +B100_retr_joint hit5%=0.57 hit10%=0.85 med_err=0.041 inval=0.03 +B100_retr_plant_first hit5%=0.42 hit10%=0.52 med_err=0.090 inval=0.07 +B100_random_prior hit5%=0.75 hit10%=0.93 med_err=0.028 inval=0.00 +B100_retr15+NM hit5%=0.78 hit10%=0.87 med_err=0.003 inval=0.07 +B100_NM_cold hit5%=0.38 hit10%=0.43 med_err=0.266 inval=0.37 +B100_diff15+NM hit5%=0.82 hit10%=0.83 med_err=0.007 inval=0.07 diff --git a/results/inverse_design/log_o2_k4_s1.txt b/results/inverse_design/log_o2_k4_s1.txt new file mode 100644 index 0000000000000000000000000000000000000000..459874b23e0ce9b6bd3a641e24c043c2c91ad52c --- /dev/null +++ b/results/inverse_design/log_o2_k4_s1.txt @@ -0,0 +1,22 @@ +bank rows 29199 plants 1397 rows/plant 20.901216893342877 corner frac 0.22120620569197574 +ep0 loss 1.0817 3s +ep10 loss 0.1476 25s +ep20 loss 0.1204 48s +ep30 loss 0.1124 72s +ep40 loss 0.1073 96s +ep50 loss 0.1040 120s +train s 141.2127387523651 +n requests 59 +sampling s 40.066301584243774 +eval s 91.19789481163025 +B15_diffusion hit5%=0.17 hit10%=0.32 med_err=0.130 inval=0.00 +B15_retr_joint hit5%=0.20 hit10%=0.42 med_err=0.135 inval=0.02 +B15_retr_plant_first hit5%=0.12 hit10%=0.29 med_err=0.231 inval=0.08 +B15_random_prior hit5%=0.31 hit10%=0.54 med_err=0.087 inval=0.00 +B100_diffusion hit5%=0.47 hit10%=0.68 med_err=0.053 inval=0.00 +B100_retr_joint hit5%=0.51 hit10%=0.80 med_err=0.049 inval=0.00 +B100_retr_plant_first hit5%=0.24 hit10%=0.61 med_err=0.087 inval=0.03 +B100_random_prior hit5%=0.69 hit10%=0.83 med_err=0.023 inval=0.00 +B100_retr15+NM hit5%=0.73 hit10%=0.90 med_err=0.010 inval=0.02 +B100_NM_cold hit5%=0.32 hit10%=0.41 med_err=0.788 inval=0.39 +B100_diff15+NM hit5%=0.78 hit10%=0.83 med_err=0.010 inval=0.00 diff --git a/results/inverse_design/log_o4_k4_s0.txt b/results/inverse_design/log_o4_k4_s0.txt new file mode 100644 index 0000000000000000000000000000000000000000..61e8f78f0f108f082e5f7f1d4d2382e7ee29f27a --- /dev/null +++ b/results/inverse_design/log_o4_k4_s0.txt @@ -0,0 +1,23 @@ +gen 1500+180 plants 328s +bank rows 24544 plants 1312 rows/plant 18.70731707317073 corner frac 0.1813070404172099 +ep0 loss 1.1258 3s +ep10 loss 0.1435 27s +ep20 loss 0.1104 51s +ep30 loss 0.1005 76s +ep40 loss 0.0963 100s +ep50 loss 0.0939 125s +train s 159.7657082080841 +n requests 60 +sampling s 66.12464332580566 +eval s 277.16771936416626 +B15_diffusion hit5%=0.07 hit10%=0.23 med_err=0.237 inval=0.05 +B15_retr_joint hit5%=0.28 hit10%=0.53 med_err=0.084 inval=0.05 +B15_retr_plant_first hit5%=0.08 hit10%=0.25 med_err=0.235 inval=0.20 +B15_random_prior hit5%=0.40 hit10%=0.57 med_err=0.080 inval=0.03 +B100_diffusion hit5%=0.37 hit10%=0.58 med_err=0.072 inval=0.00 +B100_retr_joint hit5%=0.62 hit10%=0.88 med_err=0.037 inval=0.00 +B100_retr_plant_first hit5%=0.33 hit10%=0.62 med_err=0.071 inval=0.07 +B100_random_prior hit5%=0.82 hit10%=0.92 med_err=0.018 inval=0.00 +B100_retr15+NM hit5%=0.60 hit10%=0.75 med_err=0.038 inval=0.05 +B100_NM_cold hit5%=0.28 hit10%=0.35 med_err=1.238 inval=0.47 +B100_diff15+NM hit5%=0.67 hit10%=0.77 med_err=0.026 inval=0.05 diff --git a/results/inverse_design/res_o2_k2_s0_n60_e2.json b/results/inverse_design/res_o2_k2_s0_n60_e2.json new file mode 100644 index 0000000000000000000000000000000000000000..6b23a31052a9c6917a95728766d7b8db18bdd7bf --- /dev/null +++ b/results/inverse_design/res_o2_k2_s0_n60_e2.json @@ -0,0 +1,80 @@ +{ + "cfg": { + "ORD": 2, + "K": 2, + "SHIFT": 0, + "NPL": 60, + "EP": 2, + "rows": 1040, + "train_s": 0.22450613975524902, + "n_req": 6 + }, + "res": { + "B15_diffusion": { + "hit05": 0.16666666666666666, + "hit10": 0.16666666666666666, + "med_err": 0.8243669029276106, + "invalid": 0.3333333333333333 + }, + "B15_retr_joint": { + "hit05": 0.16666666666666666, + "hit10": 0.3333333333333333, + "med_err": 0.1375035551232573, + "invalid": 0.16666666666666666 + }, + "B15_retr_plant_first": { + "hit05": 0.16666666666666666, + "hit10": 0.16666666666666666, + "med_err": 0.2534042644462727, + "invalid": 0.3333333333333333 + }, + "B15_random_prior": { + "hit05": 0.5, + "hit10": 1.0, + "med_err": 0.04094561589828882, + "invalid": 0.0 + }, + "B100_diffusion": { + "hit05": 0.5, + "hit10": 0.5, + "med_err": 0.09260954515084352, + "invalid": 0.0 + }, + "B100_retr_joint": { + "hit05": 0.6666666666666666, + "hit10": 0.8333333333333334, + "med_err": 0.021378192404253704, + "invalid": 0.0 + }, + "B100_retr_plant_first": { + "hit05": 0.6666666666666666, + "hit10": 0.8333333333333334, + "med_err": 0.021378192404253704, + "invalid": 0.16666666666666666 + }, + "B100_random_prior": { + "hit05": 0.6666666666666666, + "hit10": 1.0, + "med_err": 0.021513503362784625, + "invalid": 0.0 + }, + "B100_retr15+NM": { + "hit05": 0.8333333333333334, + "hit10": 0.8333333333333334, + "med_err": 0.008661128502538529, + "invalid": 0.16666666666666666 + }, + "B100_NM_cold": { + "hit05": 0.5, + "hit10": 0.5, + "med_err": 0.06922191665194893, + "invalid": 0.3333333333333333 + }, + "B100_diff15+NM": { + "hit05": 0.16666666666666666, + "hit10": 0.16666666666666666, + "med_err": 0.21887247089461964, + "invalid": 0.3333333333333333 + } + } +} \ No newline at end of file diff --git a/results/inverse_design/res_o2_k4_s0_n1500_e60.json b/results/inverse_design/res_o2_k4_s0_n1500_e60.json new file mode 100644 index 0000000000000000000000000000000000000000..69dcb0b199ff7f77ff4d782ee4cc4f088ee0e029 --- /dev/null +++ b/results/inverse_design/res_o2_k4_s0_n1500_e60.json @@ -0,0 +1,80 @@ +{ + "cfg": { + "ORD": 2, + "K": 4, + "SHIFT": 0, + "NPL": 1500, + "EP": 60, + "rows": 29199, + "train_s": 174.47957634925842, + "n_req": 60 + }, + "res": { + "B15_diffusion": { + "hit05": 0.21666666666666667, + "hit10": 0.48333333333333334, + "med_err": 0.11109954128406946, + "invalid": 0.06666666666666667 + }, + "B15_retr_joint": { + "hit05": 0.36666666666666664, + "hit10": 0.65, + "med_err": 0.0642909118654702, + "invalid": 0.06666666666666667 + }, + "B15_retr_plant_first": { + "hit05": 0.16666666666666666, + "hit10": 0.26666666666666666, + "med_err": 0.20785213528599678, + "invalid": 0.16666666666666666 + }, + "B15_random_prior": { + "hit05": 0.3, + "hit10": 0.5333333333333333, + "med_err": 0.07807652346467586, + "invalid": 0.0 + }, + "B100_diffusion": { + "hit05": 0.5833333333333334, + "hit10": 0.7166666666666667, + "med_err": 0.037137437610130164, + "invalid": 0.05 + }, + "B100_retr_joint": { + "hit05": 0.5666666666666667, + "hit10": 0.85, + "med_err": 0.0406469903912281, + "invalid": 0.03333333333333333 + }, + "B100_retr_plant_first": { + "hit05": 0.4166666666666667, + "hit10": 0.5166666666666667, + "med_err": 0.0898110450573486, + "invalid": 0.06666666666666667 + }, + "B100_random_prior": { + "hit05": 0.75, + "hit10": 0.9333333333333333, + "med_err": 0.02776710543199723, + "invalid": 0.0 + }, + "B100_retr15+NM": { + "hit05": 0.7833333333333333, + "hit10": 0.8666666666666667, + "med_err": 0.0031948799732121336, + "invalid": 0.06666666666666667 + }, + "B100_NM_cold": { + "hit05": 0.38333333333333336, + "hit10": 0.43333333333333335, + "med_err": 0.26590876849056677, + "invalid": 0.36666666666666664 + }, + "B100_diff15+NM": { + "hit05": 0.8166666666666667, + "hit10": 0.8333333333333334, + "med_err": 0.0068331009400988685, + "invalid": 0.06666666666666667 + } + } +} \ No newline at end of file diff --git a/results/inverse_design/res_o2_k4_s1_n1500_e60.json b/results/inverse_design/res_o2_k4_s1_n1500_e60.json new file mode 100644 index 0000000000000000000000000000000000000000..0823bee86b337c4f8d40bf74d40b77a7bfb9fc7e --- /dev/null +++ b/results/inverse_design/res_o2_k4_s1_n1500_e60.json @@ -0,0 +1,80 @@ +{ + "cfg": { + "ORD": 2, + "K": 4, + "SHIFT": 1, + "NPL": 1500, + "EP": 60, + "rows": 22740, + "train_s": 141.2127387523651, + "n_req": 59 + }, + "res": { + "B15_diffusion": { + "hit05": 0.1694915254237288, + "hit10": 0.3220338983050847, + "med_err": 0.13007828086432197, + "invalid": 0.0 + }, + "B15_retr_joint": { + "hit05": 0.2033898305084746, + "hit10": 0.423728813559322, + "med_err": 0.13510230626734976, + "invalid": 0.01694915254237288 + }, + "B15_retr_plant_first": { + "hit05": 0.11864406779661017, + "hit10": 0.288135593220339, + "med_err": 0.23060021484506787, + "invalid": 0.0847457627118644 + }, + "B15_random_prior": { + "hit05": 0.3050847457627119, + "hit10": 0.5423728813559322, + "med_err": 0.08715045383454711, + "invalid": 0.0 + }, + "B100_diffusion": { + "hit05": 0.4745762711864407, + "hit10": 0.6779661016949152, + "med_err": 0.05321863790018, + "invalid": 0.0 + }, + "B100_retr_joint": { + "hit05": 0.5084745762711864, + "hit10": 0.7966101694915254, + "med_err": 0.04886965858499126, + "invalid": 0.0 + }, + "B100_retr_plant_first": { + "hit05": 0.23728813559322035, + "hit10": 0.6101694915254238, + "med_err": 0.08749249318846997, + "invalid": 0.03389830508474576 + }, + "B100_random_prior": { + "hit05": 0.6949152542372882, + "hit10": 0.8305084745762712, + "med_err": 0.023267685319325215, + "invalid": 0.0 + }, + "B100_retr15+NM": { + "hit05": 0.7288135593220338, + "hit10": 0.8983050847457628, + "med_err": 0.009810852696813699, + "invalid": 0.01694915254237288 + }, + "B100_NM_cold": { + "hit05": 0.3220338983050847, + "hit10": 0.4067796610169492, + "med_err": 0.7878860091030313, + "invalid": 0.3898305084745763 + }, + "B100_diff15+NM": { + "hit05": 0.7796610169491526, + "hit10": 0.8305084745762712, + "med_err": 0.009683503809495786, + "invalid": 0.0 + } + } +} \ No newline at end of file diff --git a/results/inverse_design/res_o4_k4_s0_n1500_e60.json b/results/inverse_design/res_o4_k4_s0_n1500_e60.json new file mode 100644 index 0000000000000000000000000000000000000000..10b2696d6a882ab121d7619f379758d39618fb3b --- /dev/null +++ b/results/inverse_design/res_o4_k4_s0_n1500_e60.json @@ -0,0 +1,80 @@ +{ + "cfg": { + "ORD": 4, + "K": 4, + "SHIFT": 0, + "NPL": 1500, + "EP": 60, + "rows": 24544, + "train_s": 159.7657082080841, + "n_req": 60 + }, + "res": { + "B15_diffusion": { + "hit05": 0.06666666666666667, + "hit10": 0.23333333333333334, + "med_err": 0.23710563657355133, + "invalid": 0.05 + }, + "B15_retr_joint": { + "hit05": 0.2833333333333333, + "hit10": 0.5333333333333333, + "med_err": 0.08397427612306357, + "invalid": 0.05 + }, + "B15_retr_plant_first": { + "hit05": 0.08333333333333333, + "hit10": 0.25, + "med_err": 0.23480174991722047, + "invalid": 0.2 + }, + "B15_random_prior": { + "hit05": 0.4, + "hit10": 0.5666666666666667, + "med_err": 0.07960926459125814, + "invalid": 0.03333333333333333 + }, + "B100_diffusion": { + "hit05": 0.36666666666666664, + "hit10": 0.5833333333333334, + "med_err": 0.07230952125040283, + "invalid": 0.0 + }, + "B100_retr_joint": { + "hit05": 0.6166666666666667, + "hit10": 0.8833333333333333, + "med_err": 0.03739700397339013, + "invalid": 0.0 + }, + "B100_retr_plant_first": { + "hit05": 0.3333333333333333, + "hit10": 0.6166666666666667, + "med_err": 0.07071083070279865, + "invalid": 0.06666666666666667 + }, + "B100_random_prior": { + "hit05": 0.8166666666666667, + "hit10": 0.9166666666666666, + "med_err": 0.017606841009698393, + "invalid": 0.0 + }, + "B100_retr15+NM": { + "hit05": 0.6, + "hit10": 0.75, + "med_err": 0.0378771263939264, + "invalid": 0.05 + }, + "B100_NM_cold": { + "hit05": 0.2833333333333333, + "hit10": 0.35, + "med_err": 1.2384208201648002, + "invalid": 0.4666666666666667 + }, + "B100_diff15+NM": { + "hit05": 0.6666666666666666, + "hit10": 0.7666666666666667, + "med_err": 0.026459659486794435, + "invalid": 0.05 + } + } +} \ No newline at end of file diff --git a/results/pendulum/ablation_proj.log b/results/pendulum/ablation_proj.log new file mode 100644 index 0000000000000000000000000000000000000000..10bbdaee38c0d87d4b125a8e94595584abaa98c9 --- /dev/null +++ b/results/pendulum/ablation_proj.log @@ -0,0 +1,8 @@ +Sector-projection ablation — steer→min effort on the pendulum (N=20, steps=120) + + prior (no steering) effort med 10.801 stable 1.00 sector-valid 100% certified 1.00 + steer WITH projection effort med 10.489 stable 1.00 sector-valid 100% certified 1.00 + steer WITHOUT projection effort med 10.489 stable 1.00 sector-valid 100% certified 1.00 + +sector-valid = fraction of control points obeying the sign condition x·φ(x)>=0; +certified requires a valid sector, so it collapses once projection is removed. diff --git a/results/pendulum/ablation_proj_siso.log b/results/pendulum/ablation_proj_siso.log new file mode 100644 index 0000000000000000000000000000000000000000..984995981f89f6eb769516bfb57bc66906db2ed9 --- /dev/null +++ b/results/pendulum/ablation_proj_siso.log @@ -0,0 +1,3 @@ +STABLE LTI, steer->min effort (N=12): + WITH projection: sector-valid 100% stable 0.83 + WITHOUT projection: sector-valid 70% stable 0.83 diff --git a/results/pendulum/baselines_stats.txt b/results/pendulum/baselines_stats.txt new file mode 100644 index 0000000000000000000000000000000000000000..edeabcdbfd6ca15cf60c6e8157cf329ec2edb445 --- /dev/null +++ b/results/pendulum/baselines_stats.txt @@ -0,0 +1,19 @@ +Pendulum baselines + statistics (held-out N=40, seed=123) + +DIFFUSION (amortized, certified) — bootstrap 95% CIs: + regret vs per-plant oracle: median +1.0% CI [+0.9, +1.2] + ≥1 certified: 1.00 CI [1.00, 1.00] + ≥1 stabilising: 1.00 CI [1.00, 1.00] + median cost: 1.178 CI [0.802, 1.859] + +CLASSICAL per-plant LQR (on the linearisation): + stabilises the NON-LINEAR pendulum (in-region ICs): 0.95 + median cost (stable): 1.082 CI [0.717, 1.779] + per-plant design time (Riccati): 0.79 ms (NOT amortised: one solve/plant) + +CONTRAST: + - LQR is solved PER PLANT; the diffusion model amortises the whole family (2 ms/sample, + one forward pass, no per-plant solve). + - LQR uses FULL STATE u=-Kx (more info than our output sector φ) and is tuned to the + SAME quadratic cost — yet the amortised model matches per-plant-optimal to +1% and is + ABSOLUTE-STABILITY CERTIFIED in-region, which the linearisation-based LQR is not. diff --git a/results/pendulum/basin.txt b/results/pendulum/basin.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d3292fd85ca0666ff9280d96aa6f72a3ac8d51c --- /dev/null +++ b/results/pendulum/basin.txt @@ -0,0 +1,19 @@ +certified region |theta|<=0.6 rad = 34 deg +sweep theta0 (omega0=0), duration 12.0s, converged iff final_energy<0.02 + plant (l,m,b) | 17d 34d 46d 57d 69d 86d 103d 115d 138d 160d 172d + (1.13,1.11,0.24) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=92 + (0.73,0.57,0.26) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=28 + (0.51,1.04,0.24) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=38 + (0.97,0.57,0.10) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=40 + (0.75,0.70,0.16) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=38 + (1.05,1.20,0.24) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=94 + (1.12,1.19,0.08) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=98 + (0.66,0.85,0.03) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=41 + (0.54,0.76,0.15) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=29 + (1.42,0.87,0.16) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=91 + (1.00,0.52,0.02) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=37 + (0.69,0.92,0.08) | OK OK OK OK OK OK OK OK OK OK OK edge~172d peak|u|=46 + +basin edge over 12 plants: median 172d, min 172d, max 172d (certified region = 34d) +fraction with basin past the certified 0.6 rad: 1.00 +peak commanded torque ranges 28..98 (no actuator bound); recovery is high-authority DIRECT regulation, not swing-up. diff --git a/results/pendulum/basin_torque.txt b/results/pendulum/basin_torque.txt new file mode 100644 index 0000000000000000000000000000000000000000..d4c39d3a793868ce55061b7d165815924532bd4e --- /dev/null +++ b/results/pendulum/basin_torque.txt @@ -0,0 +1,8 @@ +Torque-bounded basin — plant l=1.125 m=1.107 holding torque mgl=12.22 +grid theta0 in [-pi,pi], omega0 in [-6,6] (6561 ICs), dur 10.0s + $|u|\leq0.4\,mgl$ : converged fraction of grid = 0.432 + $|u|\leq0.6\,mgl$ : converged fraction of grid = 0.609 + $|u|\leq0.9\,mgl$ : converged fraction of grid = 0.914 + unbounded : converged fraction of grid = 1.000 + +basin shrinks monotonically as the actuator saturates; the unbounded case (no |u| limit) is why raw stabilisation looks easy. diff --git a/results/pendulum/cbar_eval.json b/results/pendulum/cbar_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..7813a49d70d732f34fb78e9d9c46524bbf6aa3c4 --- /dev/null +++ b/results/pendulum/cbar_eval.json @@ -0,0 +1,68 @@ +{ + "0.7": { + "oracle_found": 40, + "cond_singleshot": { + "spec_met": 0.95, + "certified": 0.85, + "energy_regret": 8.246473273131759, + "n": 34 + }, + "cond_bestofK": { + "spec_met": 1.0, + "certified": 1.0, + "energy_regret": 0.8267481140780466, + "n": 40, + "K": 8 + }, + "posthoc": { + "spec_met": 0.375, + "certified": 0.375, + "energy_regret": 0.18879793163140582, + "n": 15 + } + }, + "0.8": { + "oracle_found": 40, + "cond_singleshot": { + "spec_met": 0.9, + "certified": 0.875, + "energy_regret": 6.167389800836868, + "n": 35 + }, + "cond_bestofK": { + "spec_met": 1.0, + "certified": 1.0, + "energy_regret": 0.6647373817862725, + "n": 40, + "K": 8 + }, + "posthoc": { + "spec_met": 0.8, + "certified": 0.8, + "energy_regret": 0.04796385021600549, + "n": 32 + } + }, + "0.9": { + "oracle_found": 40, + "cond_singleshot": { + "spec_met": 0.85, + "certified": 0.85, + "energy_regret": 5.221356699653172, + "n": 34 + }, + "cond_bestofK": { + "spec_met": 1.0, + "certified": 1.0, + "energy_regret": 1.511379904754083, + "n": 40, + "K": 8 + }, + "posthoc": { + "spec_met": 1.0, + "certified": 1.0, + "energy_regret": 0.05018168323100235, + "n": 40 + } + } +} \ No newline at end of file diff --git a/results/pendulum/constrained.txt b/results/pendulum/constrained.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae684829bf8d4b6f0488cf6d4811949afe0743c1 --- /dev/null +++ b/results/pendulum/constrained.txt @@ -0,0 +1,20 @@ +Constrained optimal control on the pendulum (held-out N=40, tracking spec ISE ≤ 0.8× prior) + +PRIOR (spec-unaware): median ISE = 0.0333, median ∫u² = 6.730 (ISE = 1.0× ⇒ violates the 0.8× spec) +STEERED toward min ∫u² s.t. ISE ≤ 0.8× prior (barrier λ=200.0): + meets spec: 0.95 median ISE = 0.0272 median ∫u² = 7.247 certified 1.00 + energy used vs prior (the cost of the tighter spec): +8% + ENERGY REGRET vs per-plant constrained oracle: median +0.1% (n=38) + +energy-vs-spec constrained-optimal frontier (steered): + ρ median ISE_max median ∫u² (meeting spec) meets certified + 0.70 0.0233 4.988 0.50 0.88 + 0.80 0.0266 6.786 0.95 1.00 + 0.90 0.0299 6.924 1.00 1.00 + 1.00 0.0333 6.708 1.00 1.00 + +INTERPRETATION: a non-quadratic, hard-CONSTRAINED optimal-control problem solved through +the differentiable non-linear pendulum — minimise control energy subject to a tracking +spec. The spec-unaware prior violates the tighter spec; steering meets it at the minimum +energy (near the per-plant constrained optimum), staying certified; a tighter spec costs +more energy (the constrained-optimal frontier). diff --git a/results/pendulum/constrained_full.log b/results/pendulum/constrained_full.log new file mode 100644 index 0000000000000000000000000000000000000000..02e154114139591bb9f93f60c0bc52d361ce1e57 --- /dev/null +++ b/results/pendulum/constrained_full.log @@ -0,0 +1,24 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/pendulum_lmi.py:65: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) + +=== CONSTRAINED OC === +Constrained optimal control on the pendulum (held-out N=40, tracking spec ISE ≤ 0.8× prior) + +PRIOR (spec-unaware): median ISE = 0.0333, median ∫u² = 6.730 (ISE = 1.0× ⇒ violates the 0.8× spec) +STEERED toward min ∫u² s.t. ISE ≤ 0.8× prior (barrier λ=200.0): + meets spec: 0.95 median ISE = 0.0272 median ∫u² = 7.247 certified 1.00 + energy used vs prior (the cost of the tighter spec): +8% + ENERGY REGRET vs per-plant constrained oracle: median +0.1% (n=38) + +energy-vs-spec constrained-optimal frontier (steered): + ρ median ISE_max median ∫u² (meeting spec) meets certified + 0.70 0.0233 4.988 0.50 0.88 + 0.80 0.0266 6.786 0.95 1.00 + 0.90 0.0299 6.924 1.00 1.00 + 1.00 0.0333 6.708 1.00 1.00 + +INTERPRETATION: a non-quadratic, hard-CONSTRAINED optimal-control problem solved through +the differentiable non-linear pendulum — minimise control energy subject to a tracking +spec. The spec-unaware prior violates the tighter spec; steering meets it at the minimum +energy (near the per-plant constrained optimum), staying certified; a tighter spec costs +more energy (the constrained-optimal frontier). diff --git a/results/pendulum/constrained_smoke.log b/results/pendulum/constrained_smoke.log new file mode 100644 index 0000000000000000000000000000000000000000..675e4db1186d32c696fb6500db85a1a173d2a88f --- /dev/null +++ b/results/pendulum/constrained_smoke.log @@ -0,0 +1,23 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/pendulum_lmi.py:65: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) + +=== CONSTRAINED OC === +Constrained optimal control on the pendulum (held-out N=10, cap θ≤0.24) + +PRIOR (quadratic-trained, constraint-unaware): + satisfies |θ|≤0.24: 0.30 median peak |θ| = 0.272 median effort 18.557 +STEERED toward min ∫u² s.t. |θ|≤0.24 (barrier λ=10000.0): + satisfies: 0.30 median peak |θ| = 0.264 median effort 18.827 certified 1.00 + effort REGRET vs per-plant constrained oracle: median +0.5% (n=3) + +effort-vs-cap constrained-optimal frontier (steered): + θ_cap mean effort (satisfying) satisfied + 0.20 nan 0.00 + 0.24 7.831 0.30 + 0.28 21.301 1.00 + 0.32 21.227 1.00 + +INTERPRETATION: a non-quadratic, hard-state-constrained optimal-control problem solved +through the differentiable non-linear pendulum: the constraint-unaware prior violates the +cap; steering enforces it at minimum effort (near the per-plant constrained optimum), +certified; tighter caps cost more effort (the constrained-optimal frontier). diff --git a/results/pendulum/constrained_smoke2.log b/results/pendulum/constrained_smoke2.log new file mode 100644 index 0000000000000000000000000000000000000000..86ba62ebc130c83b65ba65afda2da9c06af0b500 --- /dev/null +++ b/results/pendulum/constrained_smoke2.log @@ -0,0 +1,27 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/pendulum_lmi.py:65: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) +/home/cercola/miniconda3/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:3860: RuntimeWarning: Mean of empty slice. + return _methods._mean(a, axis=axis, dtype=dtype, +/home/cercola/miniconda3/lib/python3.13/site-packages/numpy/_core/_methods.py:145: RuntimeWarning: invalid value encountered in scalar divide + ret = ret.dtype.type(ret / rcount) + +=== CONSTRAINED OC === +Constrained optimal control on the pendulum (held-out N=10, actuator |u|≤2.0) + +PRIOR (quadratic-trained, constraint-unaware): + satisfies |u|≤2.0: 0.00 median peak |u| = 12.460 median ISE 0.0366 +STEERED toward min ISE=∫θ² s.t. |u|≤2.0 (barrier λ=50.0): + satisfies: 0.00 median peak |u| = 7.487 median ISE 0.1078 certified 1.00 + ISE REGRET vs per-plant constrained oracle: median +nan% (n=0) + +ISE-vs-actuator constrained-optimal frontier (steered): + u_max mean ISE (satisfying) satisfied + 1.50 nan 0.00 + 2.00 nan 0.00 + 2.50 nan 0.00 + 3.00 nan 0.00 + +INTERPRETATION: a non-quadratic, hard-ACTUATOR-constrained optimal-control problem solved +through the differentiable non-linear pendulum: the constraint-unaware prior exceeds the +actuator limit; steering enforces |u|≤u_max at minimum tracking error (near the per-plant +constrained optimum), certified; a tighter actuator costs more tracking error (the frontier). diff --git a/results/pendulum/constrained_smoke3.log b/results/pendulum/constrained_smoke3.log new file mode 100644 index 0000000000000000000000000000000000000000..4cae4294fbb65fcf259e24cd6664c38f69da2366 --- /dev/null +++ b/results/pendulum/constrained_smoke3.log @@ -0,0 +1,28 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/pendulum_lmi.py:65: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) +/home/cercola/miniconda3/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:3860: RuntimeWarning: Mean of empty slice. + return _methods._mean(a, axis=axis, dtype=dtype, +/home/cercola/miniconda3/lib/python3.13/site-packages/numpy/_core/_methods.py:145: RuntimeWarning: invalid value encountered in scalar divide + ret = ret.dtype.type(ret / rcount) + +=== CONSTRAINED OC === +Constrained optimal control on the pendulum (held-out N=10, budget E_max = 0.5× prior energy) + +PRIOR (energy-unaware): median ∫u² = 18.557, median ISE = 0.0366 (uses 1.0× energy ⇒ violates the 0.5× budget) +STEERED toward min ISE s.t. ∫u² ≤ 0.5× prior (barrier λ=30.0): + satisfies budget: 0.00 median ∫u² = 17.723 median ISE = 0.0518 certified 1.00 + ISE inflation vs prior (the cost of the budget): +42% + ISE REGRET vs per-plant constrained oracle: median +nan% (n=0) + +ISE-vs-budget constrained-optimal frontier (steered): + ρ median E_max median ISE (sat) satisfied certified + 0.35 6.495 nan 0.00 1.00 + 0.50 9.279 nan 0.00 1.00 + 0.70 12.990 nan 0.00 1.00 + 1.00 18.557 0.0365 1.00 1.00 + +INTERPRETATION: a non-quadratic, hard-CONSTRAINED optimal-control problem solved through +the differentiable non-linear pendulum — minimise tracking error subject to an energy +budget. The energy-unaware prior violates the budget; steering enforces it at the minimum +achievable tracking error (near the per-plant constrained optimum), staying certified; +tighter budgets cost more tracking error (the constrained-optimal frontier). diff --git a/results/pendulum/constrained_smoke4.log b/results/pendulum/constrained_smoke4.log new file mode 100644 index 0000000000000000000000000000000000000000..88bb71ffad79fde6d165038c207540ff8cc3bcf6 --- /dev/null +++ b/results/pendulum/constrained_smoke4.log @@ -0,0 +1,24 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/pendulum_lmi.py:65: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) + +=== CONSTRAINED OC === +Constrained optimal control on the pendulum (held-out N=10, tracking spec ISE ≤ 0.8× prior) + +PRIOR (spec-unaware): median ISE = 0.0366, median ∫u² = 18.557 (ISE = 1.0× ⇒ violates the 0.8× spec) +STEERED toward min ∫u² s.t. ISE ≤ 0.8× prior (barrier λ=200.0): + meets spec: 0.90 median ISE = 0.0299 median ∫u² = 19.732 certified 1.00 + energy used vs prior (the cost of the tighter spec): +6% + ENERGY REGRET vs per-plant constrained oracle: median +0.1% (n=9) + +energy-vs-spec constrained-optimal frontier (steered): + ρ median ISE_max median ∫u² (meeting spec) meets certified + 0.70 0.0256 10.765 0.20 1.00 + 0.80 0.0293 17.648 0.90 1.00 + 0.90 0.0330 18.989 1.00 1.00 + 1.00 0.0366 18.485 1.00 1.00 + +INTERPRETATION: a non-quadratic, hard-CONSTRAINED optimal-control problem solved through +the differentiable non-linear pendulum — minimise control energy subject to a tracking +spec. The spec-unaware prior violates the tighter spec; steering meets it at the minimum +energy (near the per-plant constrained optimum), staying certified; a tighter spec costs +more energy (the constrained-optimal frontier). diff --git a/results/pendulum/dataset_gen.log b/results/pendulum/dataset_gen.log new file mode 100644 index 0000000000000000000000000000000000000000..4563e0584aaf30c487443249aaf704231676d100 --- /dev/null +++ b/results/pendulum/dataset_gen.log @@ -0,0 +1,8 @@ + refined 512/2600 + refined 1024/2600 + refined 1536/2600 + refined 2048/2600 + refined 2560/2600 + refined 2600/2600 +certified 2600/2600 = 100% +saved -> data/pendulum_dataset.pt (N=2600, cond=3, out=11, S_scale=219.63) diff --git a/results/pendulum/demo.txt b/results/pendulum/demo.txt new file mode 100644 index 0000000000000000000000000000000000000000..5edac50ef91139bd34be62e4acf9777d91fa344e --- /dev/null +++ b/results/pendulum/demo.txt @@ -0,0 +1,11 @@ +pendulum demo figure — 6 unseen plants, 15 ICs each, seed=7 + panel 1: l=1.125 m=1.107 b=0.237 cert 8/8 gen 118.6 ms/ctrl cost 23.407 + panel 2: l=0.725 m=0.570 b=0.265 cert 8/8 gen 104.4 ms/ctrl cost 2.220 + panel 3: l=0.505 m=1.039 b=0.243 cert 8/8 gen 105.4 ms/ctrl cost 2.907 + panel 4: l=0.968 m=0.573 b=0.098 cert 8/8 gen 120.9 ms/ctrl cost 4.247 + panel 5: l=0.755 m=0.701 b=0.161 cert 8/8 gen 111.6 ms/ctrl cost 3.476 + panel 6: l=1.053 m=1.196 b=0.242 cert 8/8 gen 125.2 ms/ctrl cost 23.040 +median gen (CPU): 115.1 ms/ctrl + +SUGGESTED CAPTION: +Figure 12. Amortized synthesis on the inverted-pendulum family. Each panel is a held-out pendulum (params in title); for each, a single diffusion pass generates 8 candidate controllers and we keep the lowest-cost regionally-certified one (circle-criterion LMI). The kept controller is then rolled out in the real RK4 closed loop from a fan of 15 initial conditions spanning the certified region |theta|<=0.6 rad (shaded); every trajectory converges to the upright equilibrium theta=0. All controllers are certified; the same law stabilizes the whole IC region (no per-IC re-solve). Generation costs ~2 ms/controller on GPU (results/pendulum/eval.txt; this figure was produced on CPU at 115 ms/controller), versus 16.1 s for a per-plant multistart solve (7864x), at +1.0% median cost regret. diff --git a/results/pendulum/eval.txt b/results/pendulum/eval.txt new file mode 100644 index 0000000000000000000000000000000000000000..cf3fda586a5437a1c95ed4c71d31d0db9a5b6e7d --- /dev/null +++ b/results/pendulum/eval.txt @@ -0,0 +1,5 @@ +held-out pendulums N=40, K=8/pendulum (region |θ|≤0.6) +regional certification: per-sample 1.00, plant has ≥1 certified 1.00 +empirical stability: plant has ≥1 stabilising controller 1.00 +regret vs per-plant multistart oracle: median +1.0% mean +1.3% (n=40) +amortization: 0.65s for 320 controllers (2 ms each) vs per-plant oracle 16.1s each (→ 7864× speedup) diff --git a/results/pendulum/eval_full.log b/results/pendulum/eval_full.log new file mode 100644 index 0000000000000000000000000000000000000000..b89e59d7e0bb0cbbfbf91a86e77c28d1b6d8d8ae --- /dev/null +++ b/results/pendulum/eval_full.log @@ -0,0 +1,47 @@ + pend 0: cert 8/8 stable=True best_cost=0.572 regret=+0.3% + pend 1: cert 8/8 stable=True best_cost=0.585 regret=+0.8% + pend 2: cert 8/8 stable=True best_cost=0.376 regret=+0.5% + pend 3: cert 8/8 stable=True best_cost=1.223 regret=+1.0% + pend 4: cert 8/8 stable=True best_cost=0.279 regret=+0.6% + pend 5: cert 8/8 stable=True best_cost=1.911 regret=+1.3% + pend 6: cert 8/8 stable=True best_cost=0.719 regret=+0.5% + pend 7: cert 8/8 stable=True best_cost=0.210 regret=+0.2% + pend 8: cert 8/8 stable=True best_cost=3.167 regret=+1.8% + pend 9: cert 8/8 stable=True best_cost=3.301 regret=+2.5% + pend 10: cert 8/8 stable=True best_cost=0.579 regret=+0.6% + pend 11: cert 8/8 stable=True best_cost=0.486 regret=+0.9% + pend 12: cert 8/8 stable=True best_cost=0.560 regret=+0.0% + pend 13: cert 8/8 stable=True best_cost=0.574 regret=+0.7% + pend 14: cert 8/8 stable=True best_cost=3.181 regret=+2.1% + pend 15: cert 8/8 stable=True best_cost=1.806 regret=+0.8% + pend 16: cert 8/8 stable=True best_cost=7.575 regret=+3.4% + pend 17: cert 8/8 stable=True best_cost=1.119 regret=+1.1% + pend 18: cert 8/8 stable=True best_cost=1.554 regret=+1.8% + pend 19: cert 8/8 stable=True best_cost=3.111 regret=+1.4% + pend 20: cert 8/8 stable=True best_cost=0.843 regret=+0.9% + pend 21: cert 8/8 stable=True best_cost=1.134 regret=+1.1% + pend 22: cert 8/8 stable=True best_cost=1.954 regret=+1.1% + pend 23: cert 8/8 stable=True best_cost=1.654 regret=+1.1% + pend 24: cert 8/8 stable=True best_cost=0.558 regret=+1.0% + pend 25: cert 8/8 stable=True best_cost=0.111 regret=+0.6% + pend 26: cert 8/8 stable=True best_cost=1.493 regret=+1.0% + pend 27: cert 8/8 stable=True best_cost=3.440 regret=+1.9% + pend 28: cert 8/8 stable=True best_cost=8.958 regret=+2.6% + pend 29: cert 8/8 stable=True best_cost=1.795 regret=+0.9% + pend 30: cert 8/8 stable=True best_cost=3.646 regret=+2.4% + pend 31: cert 8/8 stable=True best_cost=3.568 regret=+2.6% + pend 32: cert 8/8 stable=True best_cost=0.627 regret=+1.0% + pend 33: cert 8/8 stable=True best_cost=4.090 regret=+3.2% + pend 34: cert 8/8 stable=True best_cost=0.646 regret=+0.0% + pend 35: cert 8/8 stable=True best_cost=0.913 regret=+1.2% + pend 36: cert 8/8 stable=True best_cost=3.871 regret=+1.9% + pend 37: cert 8/8 stable=True best_cost=5.748 regret=+2.8% + pend 38: cert 8/8 stable=True best_cost=0.885 regret=+1.1% + pend 39: cert 8/8 stable=True best_cost=1.023 regret=+0.8% + +=== PENDULUM EVAL === +held-out pendulums N=40, K=8/pendulum (region |θ|≤0.6) +regional certification: per-sample 1.00, plant has ≥1 certified 1.00 +empirical stability: plant has ≥1 stabilising controller 1.00 +regret vs per-plant multistart oracle: median +1.0% mean +1.3% (n=40) +amortization: 0.65s for 320 controllers (2 ms each) vs per-plant oracle 16.1s each (→ 7864× speedup) diff --git a/results/pendulum/fig_pareto.png b/results/pendulum/fig_pareto.png new file mode 100644 index 0000000000000000000000000000000000000000..96acffafbd732606fc02f1c729304c4b2415bab3 Binary files /dev/null and b/results/pendulum/fig_pareto.png differ diff --git a/results/pendulum/hardness.txt b/results/pendulum/hardness.txt new file mode 100644 index 0000000000000000000000000000000000000000..181659bf7cc4178bd373d48d7e633d1019fc9b5e --- /dev/null +++ b/results/pendulum/hardness.txt @@ -0,0 +1,18 @@ +inverted-pendulum family — per-instance multi-start synthesis (n_pend=20, k_inits=24, steps=100, θ0=0.25 rad) +feasible (a tuned linear φ stabilises): 20/20 + +Among FEASIBLE pendulums: +FAIL-to-stabilise rate across random inits: median 60% (mean 49%, max 75%) + -> even on STABILISABLE plants, a random init lands in the DIVERGENT basin this often. +cost spread among STABILISED inits: median 190% (max 495%) + -> multiple local minima (>1 cost mode on 20/20 feasible pendulums). +per-pendulum multi-start solve time: median 16.4s (24 inits × 100 steps). + +CONVEX CONTRAST — same multistart on LTI+sector loops (LMI-stabilisable): + fail rate median 0% vs pendulum 60%; cost spread median 0% vs pendulum 190%. + -> the LTI loop is reliably stabilised from random inits; the pendulum is not. + +INTERPRETATION: unlike LTI+sector (convex LMI), per-instance pendulum synthesis is +non-convex — random single-shot synthesis fails often and multi-start is needed +(slow/unreliable). An amortised generative prior supplies a good init / a distribution +of stabilising controllers, which is exactly the value §6r's warm-start demonstrated. diff --git a/results/pendulum/hardness_full.log b/results/pendulum/hardness_full.log new file mode 100644 index 0000000000000000000000000000000000000000..90cd895e8f779eeac54d43597a9a11e442fe035a --- /dev/null +++ b/results/pendulum/hardness_full.log @@ -0,0 +1,40 @@ + pend 0 (l=1.14,m=0.33,b=0.18): feasible=True fail 2/24=8% stab-cost spread 95% modes~4 16.9s + pend 1 (l=0.77,m=0.41,b=0.11): feasible=True fail 0/24=0% stab-cost spread 103% modes~3 16.3s + pend 2 (l=0.54,m=0.90,b=0.19): feasible=True fail 10/24=42% stab-cost spread 320% modes~6 16.5s + pend 3 (l=0.52,m=0.88,b=0.11): feasible=True fail 9/24=38% stab-cost spread 172% modes~4 16.4s + pend 4 (l=1.31,m=0.85,b=0.13): feasible=True fail 18/24=75% stab-cost spread 350% modes~5 16.2s + pend 5 (l=1.41,m=0.65,b=0.27): feasible=True fail 18/24=75% stab-cost spread 310% modes~5 16.3s + pend 6 (l=1.11,m=1.20,b=0.08): feasible=True fail 18/24=75% stab-cost spread 64% modes~4 16.3s + pend 7 (l=1.23,m=1.18,b=0.19): feasible=True fail 15/24=62% stab-cost spread 495% modes~6 16.1s + pend 8 (l=1.04,m=0.92,b=0.04): feasible=True fail 15/24=62% stab-cost spread 255% modes~6 16.3s + pend 9 (l=1.44,m=0.89,b=0.25): feasible=True fail 15/24=62% stab-cost spread 410% modes~8 16.2s + pend 10 (l=1.32,m=0.92,b=0.24): feasible=True fail 16/24=67% stab-cost spread 139% modes~5 16.5s + pend 11 (l=0.50,m=0.65,b=0.09): feasible=True fail 0/24=0% stab-cost spread 137% modes~6 17.7s + pend 12 (l=1.36,m=0.42,b=0.27): feasible=True fail 10/24=42% stab-cost spread 157% modes~5 18.1s + pend 13 (l=0.53,m=0.95,b=0.04): feasible=True fail 12/24=50% stab-cost spread 208% modes~6 18.2s + pend 14 (l=1.23,m=0.77,b=0.11): feasible=True fail 18/24=75% stab-cost spread 19% modes~3 17.0s + pend 15 (l=0.68,m=0.58,b=0.06): feasible=True fail 0/24=0% stab-cost spread 143% modes~4 17.4s + pend 16 (l=1.36,m=0.74,b=0.15): feasible=True fail 14/24=58% stab-cost spread 438% modes~7 16.9s + pend 17 (l=1.04,m=1.10,b=0.24): feasible=True fail 16/24=67% stab-cost spread 433% modes~5 16.7s + pend 18 (l=0.80,m=1.14,b=0.08): feasible=True fail 13/24=54% stab-cost spread 300% modes~7 16.1s + pend 19 (l=0.92,m=0.62,b=0.03): feasible=True fail 16/24=67% stab-cost spread 43% modes~3 16.1s + +=== PENDULUM HARDNESS === +inverted-pendulum family — per-instance multi-start synthesis (n_pend=20, k_inits=24, steps=100, θ0=0.25 rad) +feasible (a tuned linear φ stabilises): 20/20 + +Among FEASIBLE pendulums: +FAIL-to-stabilise rate across random inits: median 60% (mean 49%, max 75%) + -> even on STABILISABLE plants, a random init lands in the DIVERGENT basin this often. +cost spread among STABILISED inits: median 190% (max 495%) + -> multiple local minima (>1 cost mode on 20/20 feasible pendulums). +per-pendulum multi-start solve time: median 16.4s (24 inits × 100 steps). + +CONVEX CONTRAST — same multistart on LTI+sector loops (LMI-stabilisable): + fail rate median 0% vs pendulum 60%; cost spread median 0% vs pendulum 190%. + -> the LTI loop is reliably stabilised from random inits; the pendulum is not. + +INTERPRETATION: unlike LTI+sector (convex LMI), per-instance pendulum synthesis is +non-convex — random single-shot synthesis fails often and multi-start is needed +(slow/unreliable). An amortised generative prior supplies a good init / a distribution +of stabilising controllers, which is exactly the value §6r's warm-start demonstrated. diff --git a/results/pendulum/k_sweep_pendulum.txt b/results/pendulum/k_sweep_pendulum.txt new file mode 100644 index 0000000000000000000000000000000000000000..a8f0f63e42b1b5a643e73b4352a62cb24e58ea58 --- /dev/null +++ b/results/pendulum/k_sweep_pendulum.txt @@ -0,0 +1,10 @@ +=== K-sweep PENDULUM (N=24, seed 123, gen 258.4 ms/ctrl, per-sample stab 1.000 cert 1.000) === + K >=1 stable >=1 CERTIFIED gen time/plant + 1 1.000 1.000 0.26 s + 2 1.000 1.000 0.52 s + 4 1.000 1.000 1.03 s + 8 1.000 1.000 2.07 s + 16 1.000 1.000 4.13 s + +contrast: SISO needed K~32 to reach 1.0 (mixed dataset, uncertified); the pendulum + (cost-optimal certified targets) saturates at ~1.0 -- and with a VALID certificate. diff --git a/results/pendulum/metric_steering.txt b/results/pendulum/metric_steering.txt new file mode 100644 index 0000000000000000000000000000000000000000..387b63422d589265ab76e9aab39494d867109e7e --- /dev/null +++ b/results/pendulum/metric_steering.txt @@ -0,0 +1,13 @@ +pendulum metric steering (held-out N=40, ICs=3, steer steps=150) + + set control_e ise settling peak_devi peak_cont stable cert + prior 2.140 0.020 1.195 0.753 2.925 1.00 1.00 +steer→control_effort 2.063 0.022 1.261 0.756 2.549 1.00 0.95 + steer→ise 2.672 0.015 1.057 0.745 4.591 1.00 0.33 + steer→settling 2.639 0.015 1.060 0.745 4.499 1.00 0.40 +steer→peak_deviation 2.684 0.015 1.056 0.745 4.620 1.00 0.33 + +steer→control_effort: target -3.6% vs prior (2.140→2.063) cert 0.95 +steer→ise : target -24.8% vs prior (0.020→0.015) cert 0.33 +steer→settling : target -11.2% vs prior (1.195→1.060) cert 0.40 +steer→peak_deviation: target -1.0% vs prior (0.753→0.745) cert 0.33 diff --git a/results/pendulum/steer_torque.txt b/results/pendulum/steer_torque.txt new file mode 100644 index 0000000000000000000000000000000000000000..9f23588a0fff8f38749235ec127ae7ccafe9f84f --- /dev/null +++ b/results/pendulum/steer_torque.txt @@ -0,0 +1,4 @@ +Steer-under-bound (no retrain, straight-through clip) — plant l=1.125 m=1.107 mgl=12.22 u_max=0.6*mgl=7.33 + basin fraction: generated 0.609 -> steered 0.609 (delta +0.000) + in-region bounded cost: generated 19.245 -> steered 19.646 +NEGATIVE RESULT: local test-time steering neither enlarges the basin nor lowers cost under a hard actuator bound (saturated objective is non-smooth, cf. non-convexity §4.5). The bound must enter synthesis (retraining), not test-time patching. diff --git a/results/pendulum/table_pareto.md b/results/pendulum/table_pareto.md new file mode 100644 index 0000000000000000000000000000000000000000..5b14c47d4cae97ed2f15282a35298b61f925252c --- /dev/null +++ b/results/pendulum/table_pareto.md @@ -0,0 +1,9 @@ +**Table (§4.8).** Cost-family ρ-sweep on the pendulum. The ρ=1-trained prior is steered at inference to each ρ (J_ρ=∫q(y)+ρu², q=θ²+0.1ω²) and scored against a per-plant numerical oracle (re-optimized from a stabilising linear warm-start, 200 steps). Held-out N=40, 3 ICs, steer=60 steps, seed 321. Prior (unguided) certified fraction = 1.00. + +| ρ | steered regret vs oracle | prior regret vs oracle | certified (steered) | certified (oracle) | +|---|---|---|---|---| +| 0.1 | +0.08% | +0.62% | 1.00 | 1.00 | +| 0.3 | +0.08% | +1.10% | 1.00 | 1.00 | +| 1 | +0.10% | +1.55% | 1.00 | 1.00 | +| 3 | +0.11% | +1.70% | 1.00 | 1.00 | +| 10 | +0.12% | +1.78% | 1.00 | 1.00 | diff --git a/results/pendulum/timing.txt b/results/pendulum/timing.txt new file mode 100644 index 0000000000000000000000000000000000000000..79700d75513e388a03bda455990826177e980356 --- /dev/null +++ b/results/pendulum/timing.txt @@ -0,0 +1,18 @@ +Per-mode timing — pendulum, N=20 plants, K=8, steer_steps=150, oracle=16×100 (gen on CUDA, LMI/steer/oracle on CPU) + + stage costs: + raw generation 2.59 ms / controller ( 20.7 ms / plant for K=8) + certification (LMI) 5.85 ms / controller + steering (warm, 150 steps) 1101.4 ms / plant / objective + cold per-plant oracle 15684.9 ms / plant + + PIPELINE (per plant) what you get time vs oracle + 1. raw generation (K) K candidate controllers 20.7 ms 757× + 2. + certification a certified controller 26.6 ms 591× + 3. + steering (1 objective) optimal for a chosen objective 1127.9 ms 13.9× + 4. cold per-plant oracle the per-plant optimum 15684.9 ms 1× + + Reading: unguided synthesis (modes 1–2) is ~591× faster than a cold per-plant solve; + the STEERED capabilities (mode 3) cost more but are still ~14× faster than cold-solving + the objective from scratch, because the diffusion sample is a warm start (§4.4: ~7× fewer + steps) and one model is reused across the whole plant family. diff --git a/results/pendulum/timing_fam_smoke.log b/results/pendulum/timing_fam_smoke.log new file mode 100644 index 0000000000000000000000000000000000000000..78f5ea19b1136e8e9c861e876da244ecae45797e --- /dev/null +++ b/results/pendulum/timing_fam_smoke.log @@ -0,0 +1,15 @@ +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/build_dataset.py:104: FutureWarning: + You didn't specify the order of the reshape expression. The default order + used in CVXPY is Fortran ('F') order. This default will change to match NumPy's + default order ('C') in a future version of CVXPY. + To suppress this warning, please specify the order explicitly. + + blk22_mat = cp.reshape(blk22, (1,1)) +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/build_dataset.py:121: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) +/home/cercola/matte_server/DiffusionControl/nonlinear/data_generation/mimo_lmi.py:29: UserWarning: Solution may be inaccurate. Try another solver, adjusting the solver settings, or solve with verbose=True for more information. + prob.solve(solver=cp.SCS, verbose=False) +SISO raw 183.7 +cert 1092.8 +steer 40005.6 oracle 22919.2 ms/plant | synth 21× steer 0.6× +MIMO 2×2 raw 113.8 +cert 1823.8 +steer 16992.5 oracle 22789.4 ms/plant | synth 12× steer 1.3× +pendulum raw 20.7 +cert 26.6 +steer 1127.9 oracle 15684.9 ms/plant | synth 591× steer 13.9× +saved -> results/pendulum/timing_families.npz diff --git a/results/pendulum/timing_run.log b/results/pendulum/timing_run.log new file mode 100644 index 0000000000000000000000000000000000000000..8ece84b1aadac920002e98d300480dc6efcb8f6c --- /dev/null +++ b/results/pendulum/timing_run.log @@ -0,0 +1,20 @@ + +=== TIMING ANALYSIS === +Per-mode timing — pendulum, N=20 plants, K=8, steer_steps=150, oracle=16×100 (gen on CUDA, LMI/steer/oracle on CPU) + + stage costs: + raw generation 2.59 ms / controller ( 20.7 ms / plant for K=8) + certification (LMI) 5.85 ms / controller + steering (warm, 150 steps) 1101.4 ms / plant / objective + cold per-plant oracle 15684.9 ms / plant + + PIPELINE (per plant) what you get time vs oracle + 1. raw generation (K) K candidate controllers 20.7 ms 757× + 2. + certification a certified controller 26.6 ms 591× + 3. + steering (1 objective) optimal for a chosen objective 1127.9 ms 13.9× + 4. cold per-plant oracle the per-plant optimum 15684.9 ms 1× + + Reading: unguided synthesis (modes 1–2) is ~591× faster than a cold per-plant solve; + the STEERED capabilities (mode 3) cost more but are still ~14× faster than cold-solving + the objective from scratch, because the diffusion sample is a warm start (§4.4: ~7× fewer + steps) and one model is reused across the whole plant family. diff --git a/results/pendulum/train.log b/results/pendulum/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d82bef007dc4f802090c580bf54b58d176da7bb7 --- /dev/null +++ b/results/pendulum/train.log @@ -0,0 +1,1631 @@ +wandb: Tracking run with wandb version 0.28.0 +wandb: W&B syncing is set to `offline` in this directory. Run `wandb online` or set WANDB_MODE=online to enable cloud syncing. +wandb: Run data is saved locally in /home/cercola/matte_server/DiffusionControl/wandb/offline-run-20260626_223209-bl58c6p4 +wandb: View this run in the terminal with `wandb leet` +device cuda | N=2600 cond=3 out=11 | hidden=256 layers=5 epochs=800 + Epoch 1/800: 0%| | 0/10 [00:00=2: 86.3%, >=4: 27.0%, ==8: 0.2% +share of tier rows that are repaired: 65.2%; plants whose tier has a repaired row: 88.0% +certified finals before repair per plant (of 32): mean 7.8 diff --git a/results/pendulum_multi/multi_stats.txt b/results/pendulum_multi/multi_stats.txt new file mode 100644 index 0000000000000000000000000000000000000000..d09a66ec4336ee4dabc16b90aefb36444a4e0028 --- /dev/null +++ b/results/pendulum_multi/multi_stats.txt @@ -0,0 +1,12 @@ +multi-solution pendulum dataset: 2600 train plants (the original 2600), 32 starts x 160 steps x 6 label ICs per plant +S_scale: original 219.63, new 238.23 +plants with >=1 certified solution in the +5% pool: 2600/2600; rows: tier(<=+2%,<=8) 2611, pool(<=+5%,<=16) 3113 +stable starts per plant (of 32): mean 25.1, median 26, min 10 +distinct certified solutions within +2% (<=8, rel dist>0.10): mean 1.00, median 1, p10/p90 1/1; share with >=2: 4.3%, >=4: 0.0%, ==8: 0.0% +distinct certified within +5% (<=16, rel dist>0.10): mean 1.20, median 1, p10/p90 1/2; share with >=2: 18.7%, >=4: 0.0%, ==8: 0.0% +distinct in +5% pool at rel-dist>0.05: mean 1.17, share>=2 16% (300 plants subsample) +distinct in +5% pool at rel-dist>0.1: mean 1.17, share>=2 16% (300 plants subsample) +distinct in +5% pool at rel-dist>0.25: mean 1.01, share>=2 1% (300 plants subsample) +distinct in +5% pool at rel-dist>0.5: mean 1.00, share>=2 0% (300 plants subsample) +old single label vs new best on the 6 label ICs: stable 100.0%, excess median 1.96%, p90 3.91%, share old within 2% of new best 54.2% +per-plant compute: see build logs (about 9-10 s/plant single core, 32 starts x 160 steps x 6 ICs, batched 4 plants) diff --git a/results/pendulum_multi/per_draw.txt b/results/pendulum_multi/per_draw.txt new file mode 100644 index 0000000000000000000000000000000000000000..1e0ff3b1ca0fa783e9807193a910f0561f5889b1 --- /dev/null +++ b/results/pendulum_multi/per_draw.txt @@ -0,0 +1,48 @@ +K=1 median regret (%) per draw [123 | 321] + retr orig 3.83 | 3.06 + retr multi-A rows 0.56 | 0.26 + retr multi-B rows 0.21 | 0.13 + retr multi-B top1 0.21 | 0.13 + shipped 3.32 | 3.20 + tuned B 0.43 | 0.34 + tuned A 0.70 | 0.51 + tuned B minus retr orig 123: -3.41 p=2.5e-11 | 321: -2.74 p=1.8e-12 + tuned B minus retr multi-A rows 123: -0.07 p=0.58 | 321: +0.15 p=0.1 + tuned B minus retr multi-B rows 123: +0.12 p=0.11 | 321: +0.19 p=0.00094 + tuned B minus retr multi-B top1 123: +0.12 p=0.11 | 321: +0.19 p=0.00094 +K=2 median regret (%) per draw [123 | 321] + retr orig 2.86 | 2.54 + retr multi-A rows 0.20 | 0.13 + retr multi-B rows 0.28 | 0.13 + retr multi-B top1 0.15 | 0.08 + shipped 3.10 | 3.05 + tuned B 0.27 | 0.31 + tuned A 0.48 | 0.55 + tuned B minus retr orig 123: -2.33 p=9.1e-12 | 321: -2.25 p=9.1e-12 + tuned B minus retr multi-A rows 123: +0.04 p=0.54 | 321: +0.17 p=0.14 + tuned B minus retr multi-B rows 123: -0.06 p=0.29 | 321: +0.13 p=0.075 + tuned B minus retr multi-B top1 123: +0.11 p=0.064 | 321: +0.18 p=0.0041 +K=4 median regret (%) per draw [123 | 321] + retr orig 1.43 | 1.59 + retr multi-A rows 0.16 | 0.13 + retr multi-B rows 0.32 | 0.13 + retr multi-B top1 0.15 | 0.13 + shipped 2.95 | 2.90 + tuned B 0.28 | 0.40 + tuned A 0.37 | 0.51 + tuned B minus retr orig 123: -1.20 p=7.8e-09 | 321: -1.26 p=3.1e-10 + tuned B minus retr multi-A rows 123: +0.08 p=0.023 | 321: +0.24 p=0.00028 + tuned B minus retr multi-B rows 123: -0.12 p=0.045 | 321: +0.17 p=0.038 + tuned B minus retr multi-B top1 123: +0.09 p=0.036 | 321: +0.23 p=0.0056 +K=8 median regret (%) per draw [123 | 321] + retr orig 0.92 | 0.92 + retr multi-A rows 0.14 | 0.10 + retr multi-B rows 0.22 | 0.15 + retr multi-B top1 0.14 | 0.15 + shipped 2.83 | 2.74 + tuned B 0.24 | 0.37 + tuned A 0.51 | 0.53 + tuned B minus retr orig 123: -0.64 p=1.4e-08 | 321: -0.50 p=1.7e-06 + tuned B minus retr multi-A rows 123: +0.16 p=3.5e-05 | 321: +0.33 p=4.2e-06 + tuned B minus retr multi-B rows 123: +0.01 p=0.27 | 321: +0.14 p=0.1 + tuned B minus retr multi-B top1 123: +0.12 p=0.014 | 321: +0.25 p=0.022 diff --git a/results/pendulum_multi/stats_A.txt b/results/pendulum_multi/stats_A.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e1d97e9b6bfacb14bc06470e205c277dc740494 --- /dev/null +++ b/results/pendulum_multi/stats_A.txt @@ -0,0 +1,85 @@ +=== K=1 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 3.60 [+2.92,+4.03] 5.57 3.70 1.64 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +inverse-distance blend, original bank 100.0 100.0 100.0 3.29 [+3.00,+3.63] 5.01 3.42 1.54 +tunedA 100.0 98.8 98.8 0.62 [+0.47,+0.81] 1.73 0.91 0.77 +shipped 100.0 100.0 100.0 3.27 [+3.12,+3.50] 5.22 3.50 1.54 + +=== K=2 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 2.68 [+2.46,+2.99] 4.53 2.80 1.12 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +inverse-distance blend, original bank 100.0 100.0 100.0 2.94 [+2.68,+3.21] 4.38 2.86 1.30 +tunedA 100.0 98.9 99.6 0.52 [+0.39,+0.69] 1.74 0.72 0.47 +shipped 100.0 100.0 100.0 3.09 [+2.87,+3.26] 5.02 3.19 1.27 + +=== K=4 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 1.52 [+1.31,+1.85] 2.98 1.69 0.56 +retrieval, multi bank (K nearest rows) 100.0 98.8 100.0 0.13 [+0.09,+0.20] 0.54 0.15 0.26 +retrieval, multi bank best-per-plant 100.0 98.8 100.0 0.13 [+0.09,+0.20] 0.54 0.15 0.26 +inverse-distance blend, original bank 100.0 100.0 100.0 1.57 [+1.34,+1.89] 3.39 1.83 0.59 +tunedA 100.0 99.0 99.9 0.49 [+0.36,+0.56] 1.53 0.66 0.27 +shipped 100.0 100.0 100.0 2.93 [+2.57,+3.12] 4.75 3.00 1.11 + +=== K=8 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 0.92 [+0.79,+1.15] 1.92 1.09 0.32 +retrieval, multi bank (K nearest rows) 100.0 98.8 100.0 0.11 [+0.06,+0.17] 0.42 0.10 0.18 +retrieval, multi bank best-per-plant 100.0 98.8 100.0 0.12 [+0.06,+0.17] 0.42 0.10 0.18 +inverse-distance blend, original bank 100.0 100.0 100.0 0.92 [+0.80,+1.15] 1.93 1.12 0.32 +tunedA 100.0 98.6 100.0 0.52 [+0.45,+0.63] 1.29 0.66 0.17 +shipped 100.0 100.0 100.0 2.82 [+2.46,+3.00] 4.55 2.86 1.02 + +--- paired tests: tunedA minus other arm (negative = tunedA better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = tunedA only, n01 = other only) --- +K=1 vs retrieval, original bank -3.01 [-3.51,-2.36] p=1.5e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [PRIMARY] +K=1 vs retrieval, multi bank (K nearest rows) +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [PRIMARY] +K=1 vs retrieval, multi bank best-per-plant +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank -2.66 [-2.86,-2.44] p=2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs shipped -2.76 [-3.03,-2.44] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank -2.05 [-2.36,-1.72] p=1.2e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +0.22 [+0.16,+0.42] p=9.3e-07 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +0.22 [+0.16,+0.43] p=1.7e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank -2.16 [-2.62,-1.94] p=2.7e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs shipped -2.62 [-2.88,-2.25] p=1.2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, original bank -0.97 [-1.25,-0.58] p=7.8e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank -1.04 [-1.30,-0.61] p=1.2e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs shipped -2.57 [-2.81,-2.10] p=1.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=8 vs retrieval, original bank -0.25 [-0.48,-0.19] p=9.8e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [PRIMARY] +K=8 vs retrieval, multi bank (K nearest rows) +0.44 [+0.34,+0.53] p=4.9e-13 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [PRIMARY] +K=8 vs retrieval, multi bank best-per-plant +0.43 [+0.33,+0.53] p=5.3e-13 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank -0.25 [-0.48,-0.19] p=7.1e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=8 vs shipped -2.25 [-2.62,-1.99] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] + +--- paired tests: shipped minus other arm (negative = shipped better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = shipped only, n01 = other only) --- +K=1 vs retrieval, original bank -0.19 [-0.65,+0.42] p=0.28 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank (K nearest rows) +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank best-per-plant +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank +0.24 [-0.28,+0.37] p=0.4 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs tunedA +2.76 [+2.44,+3.03] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank +0.68 [+0.15,+0.93] p=0.015 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +2.92 [+2.59,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +2.88 [+2.53,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank +0.34 [+0.06,+0.76] p=0.016 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs tunedA +2.62 [+2.25,+2.88] p=1.2e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs retrieval, original bank +1.29 [+0.96,+1.62] p=1.4e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank +1.18 [+0.87,+1.58] p=2e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs tunedA +2.57 [+2.10,+2.81] p=1.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs retrieval, original bank +1.81 [+1.46,+1.95] p=1.8e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank (K nearest rows) +2.59 [+2.25,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs retrieval, multi bank best-per-plant +2.59 [+2.20,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank +1.78 [+1.42,+1.94] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs tunedA +2.25 [+1.99,+2.62] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] + +Holm-corrected primary tests (alpha=0.05): + tunedA vs retr_orig K=1: p=1.46e-13, Holm threshold 0.0125 -> reject + tunedA vs retr_multi_rows K=8: p=4.9e-13, Holm threshold 0.0167 -> reject + tunedA vs retr_orig K=8: p=9.77e-06, Holm threshold 0.0250 -> reject + tunedA vs retr_multi_rows K=1: p=0.00371, Holm threshold 0.0500 -> reject diff --git a/results/pendulum_multi/stats_all.txt b/results/pendulum_multi/stats_all.txt new file mode 100644 index 0000000000000000000000000000000000000000..2ed18fa1685dce931d13faede613f4d8d073de35 --- /dev/null +++ b/results/pendulum_multi/stats_all.txt @@ -0,0 +1,155 @@ +=== K=1 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 3.60 [+2.92,+4.03] 5.57 3.70 1.64 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +inverse-distance blend, original bank 100.0 100.0 100.0 3.29 [+3.00,+3.63] 5.01 3.42 1.54 +retrieval, multi-B bank (K nearest rows) 100.0 100.0 100.0 0.15 [+0.10,+0.32] 0.97 0.33 0.47 +retrieval, multi-B bank best-per-plant 100.0 100.0 100.0 0.15 [+0.10,+0.32] 0.97 0.33 0.47 +tunedB 100.0 88.2 88.2 0.37 [+0.29,+0.51] 1.18 0.57 0.51 +tunedA 100.0 98.8 98.8 0.62 [+0.47,+0.81] 1.73 0.91 0.77 +shipped 100.0 100.0 100.0 3.27 [+3.12,+3.50] 5.22 3.50 1.54 + +=== K=2 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 2.68 [+2.46,+2.99] 4.53 2.80 1.12 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +inverse-distance blend, original bank 100.0 100.0 100.0 2.94 [+2.68,+3.21] 4.38 2.86 1.30 +retrieval, multi-B bank (K nearest rows) 100.0 98.8 100.0 0.19 [+0.11,+0.38] 0.97 0.37 0.40 +retrieval, multi-B bank best-per-plant 100.0 98.8 100.0 0.13 [+0.07,+0.18] 0.72 0.29 0.37 +tunedB 100.0 91.5 98.8 0.31 [+0.20,+0.35] 0.85 0.35 0.34 +tunedA 100.0 98.9 99.6 0.52 [+0.39,+0.69] 1.74 0.72 0.47 +shipped 100.0 100.0 100.0 3.09 [+2.87,+3.26] 5.02 3.19 1.27 + +=== K=4 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 1.52 [+1.31,+1.85] 2.98 1.69 0.56 +retrieval, multi bank (K nearest rows) 100.0 98.8 100.0 0.13 [+0.09,+0.20] 0.54 0.15 0.26 +retrieval, multi bank best-per-plant 100.0 98.8 100.0 0.13 [+0.09,+0.20] 0.54 0.15 0.26 +inverse-distance blend, original bank 100.0 100.0 100.0 1.57 [+1.34,+1.89] 3.39 1.83 0.59 +retrieval, multi-B bank (K nearest rows) 100.0 97.5 100.0 0.24 [+0.12,+0.40] 1.57 0.54 0.26 +retrieval, multi-B bank best-per-plant 100.0 98.8 100.0 0.14 [+0.08,+0.21] 0.74 0.36 0.23 +tunedB 100.0 90.3 99.7 0.30 [+0.25,+0.42] 0.73 0.36 0.20 +tunedA 100.0 99.0 99.9 0.49 [+0.36,+0.56] 1.53 0.66 0.27 +shipped 100.0 100.0 100.0 2.93 [+2.57,+3.12] 4.75 3.00 1.11 + +=== K=8 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 0.92 [+0.79,+1.15] 1.92 1.09 0.32 +retrieval, multi bank (K nearest rows) 100.0 98.8 100.0 0.11 [+0.06,+0.17] 0.42 0.10 0.18 +retrieval, multi bank best-per-plant 100.0 98.8 100.0 0.12 [+0.06,+0.17] 0.42 0.10 0.18 +inverse-distance blend, original bank 100.0 100.0 100.0 0.92 [+0.80,+1.15] 1.93 1.12 0.32 +retrieval, multi-B bank (K nearest rows) 100.0 96.2 100.0 0.16 [+0.11,+0.29] 1.40 0.47 0.09 +retrieval, multi-B bank best-per-plant 100.0 98.8 100.0 0.15 [+0.10,+0.23] 0.77 0.36 0.12 +tunedB 100.0 88.6 100.0 0.32 [+0.21,+0.41] 0.88 0.35 0.10 +tunedA 100.0 98.6 100.0 0.52 [+0.45,+0.63] 1.29 0.66 0.17 +shipped 100.0 100.0 100.0 2.82 [+2.46,+3.00] 4.55 2.86 1.02 + +--- paired tests: tunedB minus other arm (negative = tunedB better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = tunedB only, n01 = other only) --- +K=1 vs retrieval, original bank -2.96 [-3.60,-2.44] p=1.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [PRIMARY] +K=1 vs retrieval, multi bank (K nearest rows) +0.07 [-0.03,+0.17] p=0.53 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [PRIMARY] +K=1 vs retrieval, multi bank best-per-plant +0.07 [-0.03,+0.17] p=0.53 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs inverse-distance blend, original bank -2.88 [-3.03,-2.59] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +0.18 [+0.08,+0.28] p=0.001 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +0.18 [+0.08,+0.28] p=0.001 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs tunedA -0.15 [-0.29,-0.09] p=1.8e-08 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs shipped -2.88 [-3.11,-2.61] p=8.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=2 vs retrieval, original bank -2.33 [-2.60,-1.94] p=1.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +0.08 [+0.01,+0.18] p=0.14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +0.09 [+0.02,+0.18] p=0.16 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank -2.50 [-2.89,-2.30] p=1.1e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +0.06 [-0.09,+0.16] p=0.69 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +0.15 [+0.08,+0.23] p=0.0015 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs tunedA -0.18 [-0.35,-0.10] p=3.8e-09 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs shipped -2.73 [-2.93,-2.43] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, original bank -1.20 [-1.65,-0.89] p=2.4e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +0.17 [+0.08,+0.24] p=2.6e-05 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +0.17 [+0.08,+0.24] p=2.6e-05 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank -1.28 [-1.71,-1.00] p=1.6e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +0.07 [-0.07,+0.17] p=0.88 n=80 | stable McNemar 0-0 p=1 | cert 2-2 p=1 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +0.18 [+0.08,+0.23] p=0.00053 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=4 vs tunedA -0.14 [-0.24,-0.10] p=1e-06 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=4 vs shipped -2.59 [-2.82,-2.22] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [exploratory] +K=8 vs retrieval, original bank -0.51 [-0.72,-0.28] p=1.4e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [PRIMARY] +K=8 vs retrieval, multi bank (K nearest rows) +0.19 [+0.15,+0.34] p=3e-09 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [PRIMARY] +K=8 vs retrieval, multi bank best-per-plant +0.19 [+0.15,+0.33] p=3.3e-09 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank -0.51 [-0.76,-0.29] p=1e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +0.07 [-0.00,+0.16] p=0.75 n=80 | stable McNemar 0-0 p=1 | cert 3-2 p=1 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +0.16 [+0.09,+0.25] p=0.00072 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=8 vs tunedA -0.14 [-0.27,-0.06] p=1.3e-07 n=80 | stable McNemar 0-0 p=1 | cert 1-2 p=1 [exploratory] +K=8 vs shipped -2.25 [-2.75,-2.08] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-2 p=0.5 [exploratory] + +--- paired tests: tunedA minus other arm (negative = tunedA better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = tunedA only, n01 = other only) --- +K=1 vs retrieval, original bank -3.01 [-3.51,-2.36] p=1.5e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank (K nearest rows) +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank best-per-plant +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank -2.66 [-2.86,-2.44] p=2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +0.34 [+0.22,+0.52] p=2.9e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +0.34 [+0.22,+0.52] p=2.9e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs tunedB +0.15 [+0.09,+0.29] p=1.8e-08 n=80 | stable McNemar 0-0 p=1 | cert 3-0 p=0.25 [exploratory] +K=1 vs shipped -2.76 [-3.03,-2.44] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank -2.05 [-2.36,-1.72] p=1.2e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +0.22 [+0.16,+0.42] p=9.3e-07 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +0.22 [+0.16,+0.43] p=1.7e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank -2.16 [-2.62,-1.94] p=2.7e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +0.24 [+0.10,+0.42] p=4.8e-05 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +0.38 [+0.20,+0.53] p=2.1e-08 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs tunedB +0.18 [+0.10,+0.35] p=3.8e-09 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=2 vs shipped -2.62 [-2.88,-2.25] p=1.2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, original bank -0.97 [-1.25,-0.58] p=7.8e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank -1.04 [-1.30,-0.61] p=1.2e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +0.18 [+0.04,+0.34] p=0.031 n=80 | stable McNemar 0-0 p=1 | cert 2-1 p=1 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +0.32 [+0.20,+0.42] p=5.3e-07 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=4 vs tunedB +0.14 [+0.10,+0.24] p=1e-06 n=80 | stable McNemar 0-0 p=1 | cert 2-1 p=1 [exploratory] +K=4 vs shipped -2.57 [-2.81,-2.10] p=1.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=8 vs retrieval, original bank -0.25 [-0.48,-0.19] p=9.8e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=8 vs retrieval, multi bank (K nearest rows) +0.44 [+0.34,+0.53] p=4.9e-13 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=8 vs retrieval, multi bank best-per-plant +0.43 [+0.33,+0.53] p=5.3e-13 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank -0.25 [-0.48,-0.19] p=7.1e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +0.34 [+0.11,+0.42] p=0.0038 n=80 | stable McNemar 0-0 p=1 | cert 3-1 p=0.62 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +0.37 [+0.20,+0.44] p=6.5e-09 n=80 | stable McNemar 0-0 p=1 | cert 1-1 p=1 [exploratory] +K=8 vs tunedB +0.14 [+0.06,+0.27] p=1.3e-07 n=80 | stable McNemar 0-0 p=1 | cert 2-1 p=1 [exploratory] +K=8 vs shipped -2.25 [-2.62,-1.99] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-1 p=1 [exploratory] + +--- paired tests: shipped minus other arm (negative = shipped better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = shipped only, n01 = other only) --- +K=1 vs retrieval, original bank -0.19 [-0.65,+0.42] p=0.28 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank (K nearest rows) +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank best-per-plant +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank +0.24 [-0.28,+0.37] p=0.4 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +3.10 [+2.81,+3.21] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +3.10 [+2.81,+3.21] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs tunedB +2.88 [+2.61,+3.11] p=8.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 3-0 p=0.25 [exploratory] +K=1 vs tunedA +2.76 [+2.44,+3.03] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank +0.68 [+0.15,+0.93] p=0.015 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +2.92 [+2.59,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +2.88 [+2.53,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank +0.34 [+0.06,+0.76] p=0.016 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +2.85 [+2.50,+3.04] p=9.5e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +2.92 [+2.53,+3.12] p=9.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=2 vs tunedB +2.73 [+2.43,+2.93] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=2 vs tunedA +2.62 [+2.25,+2.88] p=1.2e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs retrieval, original bank +1.29 [+0.96,+1.62] p=1.4e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank +1.18 [+0.87,+1.58] p=2e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +2.47 [+1.98,+2.68] p=2.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 2-0 p=0.5 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +2.55 [+2.23,+2.85] p=9.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=4 vs tunedB +2.59 [+2.22,+2.82] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 2-0 p=0.5 [exploratory] +K=4 vs tunedA +2.57 [+2.10,+2.81] p=1.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs retrieval, original bank +1.81 [+1.46,+1.95] p=1.8e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank (K nearest rows) +2.59 [+2.25,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs retrieval, multi bank best-per-plant +2.59 [+2.20,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank +1.78 [+1.42,+1.94] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +2.17 [+1.92,+2.58] p=3.4e-14 n=80 | stable McNemar 0-0 p=1 | cert 3-0 p=0.25 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +2.55 [+2.13,+2.78] p=1.5e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] +K=8 vs tunedB +2.25 [+2.08,+2.75] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 2-0 p=0.5 [exploratory] +K=8 vs tunedA +2.25 [+1.99,+2.62] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 1-0 p=1 [exploratory] + +Holm-corrected primary tests (alpha=0.05): + tunedB vs retr_orig K=1: p=1.28e-14, Holm threshold 0.0125 -> reject + tunedB vs retr_orig K=8: p=1.35e-11, Holm threshold 0.0167 -> reject + tunedB vs retr_multi_rows K=8: p=2.97e-09, Holm threshold 0.0250 -> reject + tunedB vs retr_multi_rows K=1: p=0.53, Holm threshold 0.0500 -> not significant diff --git a/results/pendulum_multi/stats_all_certfilter.txt b/results/pendulum_multi/stats_all_certfilter.txt new file mode 100644 index 0000000000000000000000000000000000000000..6a3c1f06ca47da3db950f5edaae8bf87ba82aa7a --- /dev/null +++ b/results/pendulum_multi/stats_all_certfilter.txt @@ -0,0 +1,155 @@ +=== K=1 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 3.60 [+2.92,+4.03] 5.57 3.70 1.64 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.36 [+0.13,+0.53] 1.61 0.58 0.66 +inverse-distance blend, original bank 100.0 100.0 100.0 3.29 [+3.00,+3.63] 5.01 3.42 1.54 +retrieval, multi-B bank (K nearest rows) 100.0 100.0 100.0 0.15 [+0.10,+0.32] 0.97 0.33 0.47 +retrieval, multi-B bank best-per-plant 100.0 100.0 100.0 0.15 [+0.10,+0.32] 0.97 0.33 0.47 +tunedB 100.0 88.2 88.2 0.37 [+0.29,+0.51] 1.18 0.57 0.51 +tunedA 100.0 98.8 98.8 0.62 [+0.47,+0.81] 1.73 0.91 0.77 +shipped 100.0 100.0 100.0 3.27 [+3.12,+3.50] 5.22 3.50 1.54 + +=== K=2 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 2.68 [+2.46,+2.99] 4.53 2.80 1.12 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.18 [+0.11,+0.31] 0.86 0.34 0.41 +inverse-distance blend, original bank 100.0 100.0 100.0 2.94 [+2.68,+3.21] 4.38 2.86 1.30 +retrieval, multi-B bank (K nearest rows) 100.0 100.0 100.0 0.19 [+0.11,+0.38] 0.97 0.37 0.40 +retrieval, multi-B bank best-per-plant 100.0 100.0 100.0 0.12 [+0.06,+0.17] 0.60 0.19 0.37 +tunedB 100.0 98.8 98.8 0.27 [+0.19,+0.35] 0.76 0.35 0.34 +tunedA 100.0 99.6 99.6 0.52 [+0.40,+0.70] 1.81 0.79 0.47 +shipped 100.0 100.0 100.0 3.09 [+2.87,+3.26] 5.02 3.19 1.27 + +=== K=4 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 1.52 [+1.31,+1.85] 2.98 1.69 0.56 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.13 [+0.09,+0.20] 0.54 0.16 0.26 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.13 [+0.09,+0.20] 0.54 0.16 0.26 +inverse-distance blend, original bank 100.0 100.0 100.0 1.57 [+1.34,+1.89] 3.39 1.83 0.59 +retrieval, multi-B bank (K nearest rows) 100.0 100.0 100.0 0.20 [+0.12,+0.37] 1.41 0.49 0.26 +retrieval, multi-B bank best-per-plant 100.0 100.0 100.0 0.13 [+0.06,+0.20] 0.70 0.26 0.23 +tunedB 100.0 99.7 99.7 0.29 [+0.25,+0.38] 0.73 0.33 0.20 +tunedA 100.0 99.9 99.9 0.49 [+0.36,+0.56] 1.53 0.67 0.27 +shipped 100.0 100.0 100.0 2.93 [+2.57,+3.12] 4.75 3.00 1.11 + +=== K=8 (N=80 plants; model rows = mean over runs; regret on the 24 fresh scoring ICs vs the fresh-IC oracle) === +arm stable% cert% anyCert% reg_med [95% CI] reg_p90 reg_mean in-sample reg_med +retrieval, original bank 100.0 100.0 100.0 0.92 [+0.79,+1.15] 1.92 1.09 0.32 +retrieval, multi bank (K nearest rows) 100.0 100.0 100.0 0.11 [+0.06,+0.17] 0.42 0.10 0.18 +retrieval, multi bank best-per-plant 100.0 100.0 100.0 0.12 [+0.06,+0.17] 0.42 0.10 0.18 +inverse-distance blend, original bank 100.0 100.0 100.0 0.92 [+0.80,+1.15] 1.93 1.12 0.32 +retrieval, multi-B bank (K nearest rows) 100.0 100.0 100.0 0.15 [+0.11,+0.28] 1.28 0.41 0.09 +retrieval, multi-B bank best-per-plant 100.0 100.0 100.0 0.14 [+0.08,+0.22] 0.74 0.26 0.12 +tunedB 100.0 100.0 100.0 0.29 [+0.21,+0.39] 0.68 0.33 0.10 +tunedA 100.0 100.0 100.0 0.52 [+0.47,+0.63] 1.29 0.65 0.17 +shipped 100.0 100.0 100.0 2.82 [+2.46,+3.00] 4.55 2.86 1.02 + +--- paired tests: tunedB minus other arm (negative = tunedB better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = tunedB only, n01 = other only) --- +K=1 vs retrieval, original bank -2.96 [-3.60,-2.44] p=1.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [PRIMARY] +K=1 vs retrieval, multi bank (K nearest rows) +0.07 [-0.03,+0.17] p=0.53 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [PRIMARY] +K=1 vs retrieval, multi bank best-per-plant +0.07 [-0.03,+0.17] p=0.53 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs inverse-distance blend, original bank -2.88 [-3.03,-2.59] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +0.18 [+0.08,+0.28] p=0.001 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +0.18 [+0.08,+0.28] p=0.001 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs tunedA -0.15 [-0.29,-0.09] p=1.8e-08 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=1 vs shipped -2.88 [-3.11,-2.61] p=8.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-3 p=0.25 [exploratory] +K=2 vs retrieval, original bank -2.31 [-2.53,-1.94] p=1.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +0.06 [+0.00,+0.17] p=0.21 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +0.07 [+0.00,+0.17] p=0.24 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank -2.50 [-2.89,-2.30] p=1.1e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +0.04 [-0.09,+0.15] p=0.67 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +0.15 [+0.08,+0.24] p=0.00024 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs tunedA -0.15 [-0.35,-0.09] p=5.8e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs shipped -2.69 [-2.93,-2.43] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, original bank -1.25 [-1.60,-0.84] p=8.2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +0.14 [+0.07,+0.22] p=4.1e-05 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +0.14 [+0.07,+0.22] p=4.1e-05 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank -1.28 [-1.70,-1.00] p=5.1e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +0.07 [+0.01,+0.18] p=0.84 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +0.17 [+0.08,+0.22] p=0.0003 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs tunedA -0.13 [-0.23,-0.09] p=4.2e-08 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs shipped -2.63 [-2.86,-2.31] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, original bank -0.52 [-0.72,-0.37] p=5.9e-12 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [PRIMARY] +K=8 vs retrieval, multi bank (K nearest rows) +0.18 [+0.09,+0.29] p=4.2e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [PRIMARY] +K=8 vs retrieval, multi bank best-per-plant +0.18 [+0.08,+0.29] p=5.2e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank -0.55 [-0.76,-0.41] p=5.1e-12 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +0.08 [+0.01,+0.18] p=0.43 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +0.13 [+0.07,+0.25] p=0.00028 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs tunedA -0.17 [-0.30,-0.12] p=9.4e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs shipped -2.31 [-2.75,-2.08] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] + +--- paired tests: tunedA minus other arm (negative = tunedA better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = tunedA only, n01 = other only) --- +K=1 vs retrieval, original bank -3.01 [-3.51,-2.36] p=1.5e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank (K nearest rows) +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank best-per-plant +0.19 [+0.11,+0.33] p=0.0037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank -2.66 [-2.86,-2.44] p=2e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +0.34 [+0.22,+0.52] p=2.9e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +0.34 [+0.22,+0.52] p=2.9e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs tunedB +0.15 [+0.09,+0.29] p=1.8e-08 n=80 | stable McNemar 0-0 p=1 | cert 3-0 p=0.25 [exploratory] +K=1 vs shipped -2.76 [-3.03,-2.44] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank -2.04 [-2.33,-1.65] p=4.6e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +0.22 [+0.16,+0.42] p=7.4e-07 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +0.22 [+0.16,+0.43] p=1.3e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank -2.13 [-2.58,-1.87] p=5.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +0.24 [+0.10,+0.42] p=3.9e-05 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +0.43 [+0.21,+0.58] p=1.9e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs tunedB +0.15 [+0.09,+0.35] p=5.8e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs shipped -2.62 [-2.88,-2.25] p=1.5e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, original bank -0.97 [-1.25,-0.58] p=1.5e-10 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +0.33 [+0.23,+0.51] p=5.3e-10 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank -1.04 [-1.30,-0.61] p=2e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +0.18 [+0.09,+0.37] p=0.006 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +0.33 [+0.22,+0.44] p=6.5e-08 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs tunedB +0.13 [+0.09,+0.23] p=4.2e-08 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs shipped -2.57 [-2.81,-2.10] p=2.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, original bank -0.23 [-0.48,-0.19] p=6.4e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank (K nearest rows) +0.44 [+0.34,+0.53] p=4.7e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank best-per-plant +0.43 [+0.33,+0.53] p=5.1e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank -0.23 [-0.48,-0.19] p=3.4e-06 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +0.35 [+0.12,+0.45] p=0.00037 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +0.39 [+0.24,+0.44] p=6.1e-10 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs tunedB +0.17 [+0.12,+0.30] p=9.4e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs shipped -2.25 [-2.62,-1.99] p=1.7e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] + +--- paired tests: shipped minus other arm (negative = shipped better); median paired diff in regret points [bootstrap 95% CI], Wilcoxon p, McNemar on majority-stable / picked-certified (n10 = shipped only, n01 = other only) --- +K=1 vs retrieval, original bank -0.19 [-0.65,+0.42] p=0.28 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank (K nearest rows) +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi bank best-per-plant +2.89 [+2.56,+3.13] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs inverse-distance blend, original bank +0.24 [-0.28,+0.37] p=0.4 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank (K nearest rows) +3.10 [+2.81,+3.21] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs retrieval, multi-B bank best-per-plant +3.10 [+2.81,+3.21] p=9.1e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=1 vs tunedB +2.88 [+2.61,+3.11] p=8.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 3-0 p=0.25 [exploratory] +K=1 vs tunedA +2.76 [+2.44,+3.03] p=1.6e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, original bank +0.68 [+0.15,+0.93] p=0.015 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank (K nearest rows) +2.92 [+2.59,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi bank best-per-plant +2.88 [+2.53,+3.12] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs inverse-distance blend, original bank +0.34 [+0.06,+0.76] p=0.016 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank (K nearest rows) +2.85 [+2.50,+3.04] p=9.5e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs retrieval, multi-B bank best-per-plant +2.94 [+2.59,+3.13] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs tunedB +2.69 [+2.43,+2.93] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=2 vs tunedA +2.62 [+2.25,+2.88] p=1.5e-13 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, original bank +1.29 [+0.96,+1.62] p=1.4e-11 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank (K nearest rows) +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi bank best-per-plant +2.70 [+2.32,+2.98] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs inverse-distance blend, original bank +1.18 [+0.87,+1.58] p=2e-09 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank (K nearest rows) +2.48 [+2.03,+2.72] p=1.3e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs retrieval, multi-B bank best-per-plant +2.60 [+2.24,+2.90] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs tunedB +2.63 [+2.31,+2.86] p=8.2e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=4 vs tunedA +2.57 [+2.10,+2.81] p=2.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, original bank +1.81 [+1.46,+1.95] p=1.8e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank (K nearest rows) +2.59 [+2.25,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi bank best-per-plant +2.59 [+2.20,+2.97] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs inverse-distance blend, original bank +1.78 [+1.42,+1.94] p=1.9e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank (K nearest rows) +2.20 [+1.95,+2.58] p=1.8e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs retrieval, multi-B bank best-per-plant +2.56 [+2.17,+2.79] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs tunedB +2.31 [+2.08,+2.75] p=7.8e-15 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] +K=8 vs tunedA +2.25 [+1.99,+2.62] p=1.7e-14 n=80 | stable McNemar 0-0 p=1 | cert 0-0 p=1 [exploratory] + +Holm-corrected primary tests (alpha=0.05): + tunedB vs retr_orig K=1: p=1.28e-14, Holm threshold 0.0125 -> reject + tunedB vs retr_orig K=8: p=5.87e-12, Holm threshold 0.0167 -> reject + tunedB vs retr_multi_rows K=8: p=4.22e-09, Holm threshold 0.0250 -> reject + tunedB vs retr_multi_rows K=1: p=0.53, Holm threshold 0.0500 -> not significant diff --git a/results/pendulum_multi/sweep_val.txt b/results/pendulum_multi/sweep_val.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca1d1a0154770998c7843f55fe1df0b31a60f646 --- /dev/null +++ b/results/pendulum_multi/sweep_val.txt @@ -0,0 +1,18 @@ +sw_w128_d0.1_s20k cfg_scale 1.0: val mean regret 0.587 +- 0.036 (5 sampler seeds) best_step 2500 +sw_w128_d0.1_s20k cfg_scale 2.0: val mean regret 0.682 +- 0.015 (5 sampler seeds) best_step 2500 +sw_w256_d0.1_s20k cfg_scale 1.0: val mean regret 0.556 +- 0.029 (5 sampler seeds) best_step 2500 +sw_w256_d0.1_s20k cfg_scale 2.0: val mean regret 0.634 +- 0.033 (5 sampler seeds) best_step 2500 +sw_w256_d0.1_s20k_pool5 cfg_scale 1.0: val mean regret 0.530 +- 0.052 (5 sampler seeds) best_step 1500 +sw_w256_d0_s20k cfg_scale 1.0: val mean regret 0.528 +- 0.032 (5 sampler seeds) best_step 2000 +sw_w256_d0.3_s20k cfg_scale 1.0: val mean regret 0.539 +- 0.047 (5 sampler seeds) best_step 3000 +sw_w256_d0.1_s5k cfg_scale 1.0: val mean regret 0.611 +- 0.024 (5 sampler seeds) best_step 3000 +sw_w256_d0.1_s60k cfg_scale 1.0: val mean regret 0.561 +- 0.032 (5 sampler seeds) best_step 3000 +sw_w512_d0.1_s20k cfg_scale 1.0: val mean regret 0.530 +- 0.056 (5 sampler seeds) best_step 1500 +sw_w128_d0.1_s20k.log DONE sw_w128_d0.1_s20k best (0.5605567273693898, 2500) 95s +sw_w256_d0.1_s20k.log DONE sw_w256_d0.1_s20k best (0.544204388101082, 2500) 130s +sw_w256_d0.1_s20k_pool5.log DONE sw_w256_d0.1_s20k_pool5 best (0.515041477742815, 1500) 118s +sw_w256_d0.1_s5k.log DONE sw_w256_d0.1_s5k best (0.5958462138236971, 3000) 91s +sw_w256_d0.1_s60k.log DONE sw_w256_d0.1_s60k best (0.556441395855245, 3000) 248s +sw_w256_d0.3_s20k.log DONE sw_w256_d0.3_s20k best (0.5363740929919999, 3000) 141s +sw_w256_d0_s20k.log DONE sw_w256_d0_s20k best (0.5284781220740893, 2000) 124s +sw_w512_d0.1_s20k.log DONE sw_w512_d0.1_s20k best (0.5570568264759682, 1500) 190s