Spaces:

JavideuS
/

Spooky

Sleeping

Spooky / quantum /tests /test_variable_scaling.py

Deploy 7d1d5a0

a115eaf verified 2 months ago

12.3 kB

"""
The build-only QUBO size measurement in
quantum.benchmark.analysis.variable_scaling.
These build real QUBOs against the committed 3x3 maps but never invoke a
solver, so they stay fast and cannot fail on qubit limits -- which is the
whole reason the module exists.
"""

import pandas as pd
import pytest

from quantum.benchmark.analysis.variable_scaling import (
    instances_from_sweep_config,
    measure_instance,
    measure_scaling,
    relative_to_naive,
    relative_to_raw,
)
from quantum.utils import preprocess as pm

MAP = "quantum/maps/synthetic/3x3/no_obs3x3"
PENALTIES = {
    "K_hot": 9, "K_adj": 4.8, "K_start": 6.5, "K_goal": 3, "K_lock": 4,
    "K_bt": 2.3, "K_tp": 1.2, "K_goal_approx": 0.7, "K_obs": 0,
    "K_crash": 2.7, "K_swap": 3,
}


@pytest.fixture(scope="module")
def rows():
    return {
        r["preprocess"]: r
        for r in measure_instance(MAP, "baseline", PENALTIES, pm.MODES)
    }


def test_every_mode_is_measurable_without_a_solver(rows):
    assert set(rows) == set(pm.MODES)
    assert all(r["error"] is None for r in rows.values())


def test_raw_is_the_unpruned_baseline(rows):
    """raw is the mode a sweep is least likely to record, and the one every
    other mode's reduction ratio is implicitly measured against."""
    raw = rows[pm.RAW]["window_variables"]
    for mode in (pm.BFS_AGGRESSIVE, pm.BFS_SAFE, pm.FULL, pm.FULL_SAFE):
        assert rows[mode]["window_variables"] <= raw, mode


def test_safe_bfs_prunes_less_than_aggressive(rows):
    """The monotone set allows waiting and revisiting, so it can only be a
    superset of the frontier -- if this ever inverts, the safe variant is not
    actually safe."""
    assert (
        rows[pm.BFS_SAFE]["window_variables"]
        > rows[pm.BFS_AGGRESSIVE]["window_variables"]
    )


def test_numeric_modes_only_differ_after_the_numeric_stage(rows):
    """full shares bfs_aggressive's BFS and full_safe shares bfs_safe's, so
    the pre-numeric counts must match exactly; only the post-numeric count
    may drop."""
    assert (
        rows[pm.FULL]["window_variables"]
        == rows[pm.BFS_AGGRESSIVE]["window_variables"]
    )
    assert (
        rows[pm.FULL_SAFE]["window_variables"] == rows[pm.BFS_SAFE]["window_variables"]
    )
    assert (
        rows[pm.FULL]["window_variables_after_numeric"]
        < rows[pm.FULL]["window_variables"]
    )
    # modes without the numeric stage must not shrink at all
    for mode in (pm.BFS_AGGRESSIVE, pm.BFS_SAFE):
        assert (
            rows[mode]["window_variables_after_numeric"]
            == rows[mode]["window_variables"]
        )


def test_encoded_variables_is_mode_independent(rows):
    """Pruning removes variables from the window, not from the problem, so the
    full encoding size is the denominator that makes instances comparable."""
    assert len({r["encoded_variables"] for r in rows.values()}) == 1


def test_a_malformed_problem_is_reported_not_raised():
    """obs5x5_hard/three_robots puts robot_2's goal on an obstacle. One bad
    instance must not abort a scan of thirty good ones."""
    out = measure_instance(
        "quantum/maps/synthetic/5x5/obs5x5_hard",
        "three_robots",
        PENALTIES,
        (pm.BFS_AGGRESSIVE,),
    )
    assert len(out) == 1
    assert "on an obstacle" in out[0]["error"]


def test_relative_to_raw_normalizes_per_instance():
    df = relative_to_raw(
        measure_scaling([(MAP, "baseline")], PENALTIES, (pm.RAW, pm.BFS_AGGRESSIVE))
    )
    raw = df[df["preprocess"] == pm.RAW].iloc[0]
    aggressive = df[df["preprocess"] == pm.BFS_AGGRESSIVE].iloc[0]
    assert raw["vs_raw"] == pytest.approx(1.0)
    assert 0 < aggressive["vs_raw"] < 1


def test_instances_come_from_the_sweep_config_verbatim():
    """The scaling table must cover exactly what the sweep runs, so the two
    outputs join on (instance_map, problem_name, preprocess)."""
    config = {
        "instances": [
            {"map": "a", "problems": ["p1", "p2"]},
            {"map": "b", "problems": ["p3"]},
        ]
    }
    assert instances_from_sweep_config(config) == [("a", "p1"), ("a", "p2"), ("b", "p3")]
    assert instances_from_sweep_config({}) == []


def test_raw_is_the_unpruned_window_not_the_whole_problem():
    """The distinction the columns exist to preserve: windowing cuts the
    encoding before any BFS runs, so attributing raw's whole gap to
    pre-processing would overstate what BFS does.
    5x5/two_robots is used rather than the 3x3 fixture because on the 3x3 the
    window (13 steps) already spans the whole horizon (12), so windowing
    reduces nothing there and raw == encoded. That is the honest edge case:
    windowing only pays off once the horizon outgrows one window.
    """
    rows = {
        r["preprocess"]: r
        for r in measure_instance(
            "quantum/maps/synthetic/5x5/obs5x5_easy",
            "two_robots",
            PENALTIES,
            (pm.RAW,),
        )
    }
    raw = rows[pm.RAW]
    assert raw["num_windows"] > 1
    assert raw["window_variables"] < raw["encoded_variables"]
    assert raw["fraction_of_encoding"] == pytest.approx(
        raw["window_variables"] / raw["encoded_variables"]
    )


def test_windowing_is_identical_across_modes(rows):
    """Modes differ only in pruning, never in how the horizon is cut, so
    vs_raw isolates pruning cleanly."""
    assert len({r["window_max_steps"] for r in rows.values()}) == 1
    assert len({r["num_windows"] for r in rows.values()}) == 1


def test_a_single_window_horizon_reduces_nothing(rows):
    """On the 3x3 the window spans the whole horizon, so raw's window is the
    full encoding -- windowing is not a free reduction, it only helps once
    the horizon exceeds one window."""
    raw = rows[pm.RAW]
    assert raw["num_windows"] == 1
    assert raw["window_variables"] == raw["encoded_variables"]


def test_peak_and_total_are_reported_separately(rows):
    """window_variables is what a qubit budget pays for (one window held at a
    time); total_window_variables is what compute pays for. Windowing pushes
    them in opposite directions, so reporting only one is misleading."""
    for mode, row in rows.items():
        assert row["num_windows"] >= 1, mode
        assert row["total_window_variables"] == (
            row["window_variables_after_numeric"] * row["num_windows"]
        ), mode


def test_windowing_gain_depends_on_the_pruning():
    """Windowing is not a free reduction and not a pure concession either.
    With no pruning it costs MORE in total, because consecutive windows
    overlap by a timestep and re-cover the same cells. With sound pruning it
    wins by a wide margin, because a monotone reachable set saturates to the
    whole free space over a long horizon and stays there, while windowing
    re-seeds it small every couple of steps and keeps BFS in the early-steps
    regime where it prunes hard.
    """
    rows = {
        r["preprocess"]: r
        for r in measure_instance(
            "quantum/maps/synthetic/5x5/obs5x5_easy",
            "two_robots",
            PENALTIES,
            (pm.RAW, pm.BFS_SAFE),
        )
    }
    raw, safe = rows[pm.RAW], rows[pm.BFS_SAFE]
    assert raw["num_windows"] > 1

    # unpruned: the overlap is pure overhead
    assert raw["total_window_variables"] > raw["full_horizon_variables"]
    assert raw["windowing_gain"] < 1

    # sound pruning: windowing more than pays for the overlap
    assert safe["total_window_variables"] < safe["full_horizon_variables"]
    assert safe["windowing_gain"] > 1


def test_full_horizon_can_be_skipped():
    """It is the only measurement that scales with the map, so a very large
    grid must be measurable without it."""
    rows = measure_instance(
        MAP, "baseline", PENALTIES, (pm.BFS_SAFE,), measure_full_horizon=False
    )
    assert rows[0]["full_horizon_variables"] is None
    assert rows[0]["windowing_gain"] is None
    assert rows[0]["window_variables"] > 0


def test_raw_windowing_never_beats_the_naive_baseline():
    """The demonstration the naive_gain column exists for.
    Windowing with no pruning can only lose: consecutive windows overlap by a
    timestep and re-cover the same cells, and nothing is removed to offset it.
    The best raw can do is break even, on an instance whose horizon fits in a
    single window -- and then it is not really windowed at all.
    """
    instances = [
        ("quantum/maps/synthetic/3x3/no_obs3x3", "baseline"),          # 1 window
        ("quantum/maps/synthetic/5x5/obs5x5_easy", "two_robots"),      # 4 windows
        ("quantum/maps/synthetic/5x5/obs5x5_easy", "three_robots"),    # 6 windows
    ]
    df = relative_to_naive(
        measure_scaling(instances, PENALTIES, (pm.RAW, pm.BFS_SAFE))
    )

    raw = df[df["preprocess"] == pm.RAW]
    assert (raw["naive_gain"] <= 1.0 + 1e-9).all(), raw[
        ["instance_map", "problem_name", "naive_gain"]
    ]
    # break-even only where there is a single window; a real split costs more
    multi = raw[raw["num_windows"] > 1]
    assert not multi.empty
    assert (multi["naive_gain"] < 1.0).all()

    # pruning is what turns windowing from a cost into a saving
    safe = df[df["preprocess"] == pm.BFS_SAFE]
    assert (safe["naive_gain"] > 1.0).all()


def test_naive_gain_is_measured_against_raw_full_horizon():
    """Not against raw's windowed count: the baseline is 'write the problem
    down and hand it over', which is one un-windowed QUBO with no pruning."""
    df = relative_to_naive(
        measure_scaling(
            [("quantum/maps/synthetic/5x5/obs5x5_easy", "two_robots")],
            PENALTIES,
            (pm.RAW, pm.BFS_SAFE),
        )
    )
    naive = df[df["preprocess"] == pm.RAW].iloc[0]["full_horizon_variables"]
    safe = df[df["preprocess"] == pm.BFS_SAFE].iloc[0]
    assert safe["naive_gain"] == pytest.approx(
        naive / safe["total_window_variables"]
    )


def test_an_oversized_window_is_skipped_not_built():
    """`raw` at 50x50 is cells x robots x t_max -- 30,000 variables for four
    robots, and the QUBO build is quadratic in pairwise terms. Building it
    stalled the whole table with no output; it is now reported as a skip."""
    rows = measure_instance(
        "quantum/maps/synthetic/50x50/no_obs50x50",
        "four_robots",
        PENALTIES,
        (pm.RAW, pm.BFS_AGGRESSIVE),
        measure_full_horizon=False,
    )
    by_mode = {r["preprocess"]: r for r in rows}
    assert "exceeds" in by_mode[pm.RAW]["error"]
    # the pruned mode on the same instance stays tiny and is measured
    assert by_mode[pm.BFS_AGGRESSIVE]["error"] is None
    assert by_mode[pm.BFS_AGGRESSIVE]["window_variables"] < 100


def test_error_rows_carry_the_full_schema():
    """relative_to_raw() reads window_variables. A narrower error row made it
    KeyError as soon as every mode for an instance failed -- which is exactly
    what a large map produces."""
    rows = measure_instance(
        "quantum/maps/synthetic/5x5/obs5x5_hard",  # malformed: goal on obstacle
        "three_robots",
        PENALTIES,
        (pm.RAW, pm.BFS_SAFE),
    )
    assert all(r["error"] is not None for r in rows)
    df = relative_to_naive(relative_to_raw(pd.DataFrame(rows)))
    assert "window_variables" in df.columns
    assert df["window_variables"].isna().all()


def test_sweep_dir_addressing_matches_the_other_analysis_clis():
    """run_aggregate/run_plots/run_report all take --sweep-dir; this one took
    only --config or --map, so pointing it at a finished sweep meant
    remembering which config produced it."""
    from quantum.benchmark.analysis.variable_scaling import (
        penalty_set_from_config,
    )

    config = {
        "instances": [{"map": "m", "problems": ["p1", "p2"]}],
        "solvers": [
            {"name": "ilp", "penalty_set": None},
            {"name": "sa", "penalty_set": "crash"},
            {"name": "qaoa", "penalty_set": "crash"},
        ],
    }
    assert instances_from_sweep_config(config) == [("m", "p1"), ("m", "p2")]
    # ILP/CBS record None and must not confuse the majority
    assert penalty_set_from_config(config) == "crash"
    # genuinely ambiguous configs get no default rather than a wrong one
    config["solvers"][1]["penalty_set"] = "swap"
    assert penalty_set_from_config(config) is None

	"""
	The build-only QUBO size measurement in
	quantum.benchmark.analysis.variable_scaling.

	These build real QUBOs against the committed 3x3 maps but never invoke a
	solver, so they stay fast and cannot fail on qubit limits -- which is the
	whole reason the module exists.
	"""

	import pandas as pd
	import pytest

	from quantum.benchmark.analysis.variable_scaling import (
	instances_from_sweep_config,
	measure_instance,
	measure_scaling,
	relative_to_naive,
	relative_to_raw,
	)
	from quantum.utils import preprocess as pm

	MAP = "quantum/maps/synthetic/3x3/no_obs3x3"
	PENALTIES = {
	"K_hot": 9, "K_adj": 4.8, "K_start": 6.5, "K_goal": 3, "K_lock": 4,
	"K_bt": 2.3, "K_tp": 1.2, "K_goal_approx": 0.7, "K_obs": 0,
	"K_crash": 2.7, "K_swap": 3,
	}


	@pytest.fixture(scope="module")
	def rows():
	return {
	r["preprocess"]: r
	for r in measure_instance(MAP, "baseline", PENALTIES, pm.MODES)
	}


	def test_every_mode_is_measurable_without_a_solver(rows):
	assert set(rows) == set(pm.MODES)
	assert all(r["error"] is None for r in rows.values())


	def test_raw_is_the_unpruned_baseline(rows):
	"""raw is the mode a sweep is least likely to record, and the one every
	other mode's reduction ratio is implicitly measured against."""
	raw = rows[pm.RAW]["window_variables"]
	for mode in (pm.BFS_AGGRESSIVE, pm.BFS_SAFE, pm.FULL, pm.FULL_SAFE):
	assert rows[mode]["window_variables"] <= raw, mode


	def test_safe_bfs_prunes_less_than_aggressive(rows):
	"""The monotone set allows waiting and revisiting, so it can only be a
	superset of the frontier -- if this ever inverts, the safe variant is not
	actually safe."""
	assert (
	rows[pm.BFS_SAFE]["window_variables"]
	> rows[pm.BFS_AGGRESSIVE]["window_variables"]
	)


	def test_numeric_modes_only_differ_after_the_numeric_stage(rows):
	"""full shares bfs_aggressive's BFS and full_safe shares bfs_safe's, so
	the pre-numeric counts must match exactly; only the post-numeric count
	may drop."""
	assert (
	rows[pm.FULL]["window_variables"]
	== rows[pm.BFS_AGGRESSIVE]["window_variables"]
	)
	assert (
	rows[pm.FULL_SAFE]["window_variables"] == rows[pm.BFS_SAFE]["window_variables"]
	)
	assert (
	rows[pm.FULL]["window_variables_after_numeric"]
	< rows[pm.FULL]["window_variables"]
	)
	# modes without the numeric stage must not shrink at all
	for mode in (pm.BFS_AGGRESSIVE, pm.BFS_SAFE):
	assert (
	rows[mode]["window_variables_after_numeric"]
	== rows[mode]["window_variables"]
	)


	def test_encoded_variables_is_mode_independent(rows):
	"""Pruning removes variables from the window, not from the problem, so the
	full encoding size is the denominator that makes instances comparable."""
	assert len({r["encoded_variables"] for r in rows.values()}) == 1


	def test_a_malformed_problem_is_reported_not_raised():
	"""obs5x5_hard/three_robots puts robot_2's goal on an obstacle. One bad
	instance must not abort a scan of thirty good ones."""
	out = measure_instance(
	"quantum/maps/synthetic/5x5/obs5x5_hard",
	"three_robots",
	PENALTIES,
	(pm.BFS_AGGRESSIVE,),
	)
	assert len(out) == 1
	assert "on an obstacle" in out[0]["error"]


	def test_relative_to_raw_normalizes_per_instance():
	df = relative_to_raw(
	measure_scaling([(MAP, "baseline")], PENALTIES, (pm.RAW, pm.BFS_AGGRESSIVE))
	)
	raw = df[df["preprocess"] == pm.RAW].iloc[0]
	aggressive = df[df["preprocess"] == pm.BFS_AGGRESSIVE].iloc[0]
	assert raw["vs_raw"] == pytest.approx(1.0)
	assert 0 < aggressive["vs_raw"] < 1


	def test_instances_come_from_the_sweep_config_verbatim():
	"""The scaling table must cover exactly what the sweep runs, so the two
	outputs join on (instance_map, problem_name, preprocess)."""
	config = {
	"instances": [
	{"map": "a", "problems": ["p1", "p2"]},
	{"map": "b", "problems": ["p3"]},
	]
	}
	assert instances_from_sweep_config(config) == [("a", "p1"), ("a", "p2"), ("b", "p3")]
	assert instances_from_sweep_config({}) == []


	def test_raw_is_the_unpruned_window_not_the_whole_problem():
	"""The distinction the columns exist to preserve: windowing cuts the
	encoding before any BFS runs, so attributing raw's whole gap to
	pre-processing would overstate what BFS does.

	5x5/two_robots is used rather than the 3x3 fixture because on the 3x3 the
	window (13 steps) already spans the whole horizon (12), so windowing
	reduces nothing there and raw == encoded. That is the honest edge case:
	windowing only pays off once the horizon outgrows one window.
	"""
	rows = {
	r["preprocess"]: r
	for r in measure_instance(
	"quantum/maps/synthetic/5x5/obs5x5_easy",
	"two_robots",
	PENALTIES,
	(pm.RAW,),
	)
	}
	raw = rows[pm.RAW]
	assert raw["num_windows"] > 1
	assert raw["window_variables"] < raw["encoded_variables"]
	assert raw["fraction_of_encoding"] == pytest.approx(
	raw["window_variables"] / raw["encoded_variables"]
	)


	def test_windowing_is_identical_across_modes(rows):
	"""Modes differ only in pruning, never in how the horizon is cut, so
	vs_raw isolates pruning cleanly."""
	assert len({r["window_max_steps"] for r in rows.values()}) == 1
	assert len({r["num_windows"] for r in rows.values()}) == 1


	def test_a_single_window_horizon_reduces_nothing(rows):
	"""On the 3x3 the window spans the whole horizon, so raw's window is the
	full encoding -- windowing is not a free reduction, it only helps once
	the horizon exceeds one window."""
	raw = rows[pm.RAW]
	assert raw["num_windows"] == 1
	assert raw["window_variables"] == raw["encoded_variables"]


	def test_peak_and_total_are_reported_separately(rows):
	"""window_variables is what a qubit budget pays for (one window held at a
	time); total_window_variables is what compute pays for. Windowing pushes
	them in opposite directions, so reporting only one is misleading."""
	for mode, row in rows.items():
	assert row["num_windows"] >= 1, mode
	assert row["total_window_variables"] == (
	row["window_variables_after_numeric"] * row["num_windows"]
	), mode


	def test_windowing_gain_depends_on_the_pruning():
	"""Windowing is not a free reduction and not a pure concession either.

	With no pruning it costs MORE in total, because consecutive windows
	overlap by a timestep and re-cover the same cells. With sound pruning it
	wins by a wide margin, because a monotone reachable set saturates to the
	whole free space over a long horizon and stays there, while windowing
	re-seeds it small every couple of steps and keeps BFS in the early-steps
	regime where it prunes hard.
	"""
	rows = {
	r["preprocess"]: r
	for r in measure_instance(
	"quantum/maps/synthetic/5x5/obs5x5_easy",
	"two_robots",
	PENALTIES,
	(pm.RAW, pm.BFS_SAFE),
	)
	}
	raw, safe = rows[pm.RAW], rows[pm.BFS_SAFE]
	assert raw["num_windows"] > 1

	# unpruned: the overlap is pure overhead
	assert raw["total_window_variables"] > raw["full_horizon_variables"]
	assert raw["windowing_gain"] < 1

	# sound pruning: windowing more than pays for the overlap
	assert safe["total_window_variables"] < safe["full_horizon_variables"]
	assert safe["windowing_gain"] > 1


	def test_full_horizon_can_be_skipped():
	"""It is the only measurement that scales with the map, so a very large
	grid must be measurable without it."""
	rows = measure_instance(
	MAP, "baseline", PENALTIES, (pm.BFS_SAFE,), measure_full_horizon=False
	)
	assert rows[0]["full_horizon_variables"] is None
	assert rows[0]["windowing_gain"] is None
	assert rows[0]["window_variables"] > 0


	def test_raw_windowing_never_beats_the_naive_baseline():
	"""The demonstration the naive_gain column exists for.

	Windowing with no pruning can only lose: consecutive windows overlap by a
	timestep and re-cover the same cells, and nothing is removed to offset it.
	The best raw can do is break even, on an instance whose horizon fits in a
	single window -- and then it is not really windowed at all.
	"""
	instances = [
	("quantum/maps/synthetic/3x3/no_obs3x3", "baseline"), # 1 window
	("quantum/maps/synthetic/5x5/obs5x5_easy", "two_robots"), # 4 windows
	("quantum/maps/synthetic/5x5/obs5x5_easy", "three_robots"), # 6 windows
	]
	df = relative_to_naive(
	measure_scaling(instances, PENALTIES, (pm.RAW, pm.BFS_SAFE))
	)

	raw = df[df["preprocess"] == pm.RAW]
	assert (raw["naive_gain"] <= 1.0 + 1e-9).all(), raw[
	["instance_map", "problem_name", "naive_gain"]
	]
	# break-even only where there is a single window; a real split costs more
	multi = raw[raw["num_windows"] > 1]
	assert not multi.empty
	assert (multi["naive_gain"] < 1.0).all()

	# pruning is what turns windowing from a cost into a saving
	safe = df[df["preprocess"] == pm.BFS_SAFE]
	assert (safe["naive_gain"] > 1.0).all()


	def test_naive_gain_is_measured_against_raw_full_horizon():
	"""Not against raw's windowed count: the baseline is 'write the problem
	down and hand it over', which is one un-windowed QUBO with no pruning."""
	df = relative_to_naive(
	measure_scaling(
	[("quantum/maps/synthetic/5x5/obs5x5_easy", "two_robots")],
	PENALTIES,
	(pm.RAW, pm.BFS_SAFE),
	)
	)
	naive = df[df["preprocess"] == pm.RAW].iloc[0]["full_horizon_variables"]
	safe = df[df["preprocess"] == pm.BFS_SAFE].iloc[0]
	assert safe["naive_gain"] == pytest.approx(
	naive / safe["total_window_variables"]
	)


	def test_an_oversized_window_is_skipped_not_built():
	"""`raw` at 50x50 is cells x robots x t_max -- 30,000 variables for four
	robots, and the QUBO build is quadratic in pairwise terms. Building it
	stalled the whole table with no output; it is now reported as a skip."""
	rows = measure_instance(
	"quantum/maps/synthetic/50x50/no_obs50x50",
	"four_robots",
	PENALTIES,
	(pm.RAW, pm.BFS_AGGRESSIVE),
	measure_full_horizon=False,
	)
	by_mode = {r["preprocess"]: r for r in rows}
	assert "exceeds" in by_mode[pm.RAW]["error"]
	# the pruned mode on the same instance stays tiny and is measured
	assert by_mode[pm.BFS_AGGRESSIVE]["error"] is None
	assert by_mode[pm.BFS_AGGRESSIVE]["window_variables"] < 100


	def test_error_rows_carry_the_full_schema():
	"""relative_to_raw() reads window_variables. A narrower error row made it
	KeyError as soon as every mode for an instance failed -- which is exactly
	what a large map produces."""
	rows = measure_instance(
	"quantum/maps/synthetic/5x5/obs5x5_hard", # malformed: goal on obstacle
	"three_robots",
	PENALTIES,
	(pm.RAW, pm.BFS_SAFE),
	)
	assert all(r["error"] is not None for r in rows)
	df = relative_to_naive(relative_to_raw(pd.DataFrame(rows)))
	assert "window_variables" in df.columns
	assert df["window_variables"].isna().all()


	def test_sweep_dir_addressing_matches_the_other_analysis_clis():
	"""run_aggregate/run_plots/run_report all take --sweep-dir; this one took
	only --config or --map, so pointing it at a finished sweep meant
	remembering which config produced it."""
	from quantum.benchmark.analysis.variable_scaling import (
	penalty_set_from_config,
	)

	config = {
	"instances": [{"map": "m", "problems": ["p1", "p2"]}],
	"solvers": [
	{"name": "ilp", "penalty_set": None},
	{"name": "sa", "penalty_set": "crash"},
	{"name": "qaoa", "penalty_set": "crash"},
	],
	}
	assert instances_from_sweep_config(config) == [("m", "p1"), ("m", "p2")]
	# ILP/CBS record None and must not confuse the majority
	assert penalty_set_from_config(config) == "crash"
	# genuinely ambiguous configs get no default rather than a wrong one
	config["solvers"][1]["penalty_set"] = "swap"
	assert penalty_set_from_config(config) is None