Blame for tests/test_environment.py

"""Environment contract tests: determinism, reward correctness, and the
failure modes the tasks are designed to catch."""

import pytest
from fastapi.testclient import TestClient

from server.app import EPISODES, app

client = TestClient(app)


@pytest.fixture(autouse=True)
def clean_episodes():
    EPISODES.clear()
    yield


def reset(task_id, seed=0, **opts):
    r = client.post(f"/api/{task_id}/reset", json={"seed": seed, **opts})
    assert r.status_code == 200
    return r.json()


def oracle(task_id):
    return client.get(f"/api/{task_id}/state").json()["state"]


def verify(task_id):
    return client.get(f"/api/{task_id}/verify").json()


def act(task_id, payload):
    return client.post(f"/api/{task_id}/action", json=payload)


# Fully seed-deterministic tasks: identical (task, seed) -> identical state,
# and instructions vary across seeds. flash-offer is deliberately excluded —
# its grading is tied to real elapsed time, so it gets its own tests below.
ALL_TASKS = ["email-triage", "settings-panel", "checkout-form", "causal-gate"]
EVERY_TASK = ALL_TASKS + ["flash-offer"]


# ---------------------------------------------------------------- determinism

@pytest.mark.parametrize("task_id", ALL_TASKS)
@pytest.mark.parametrize("seed", [0, 7, 12345])
def test_same_seed_same_episode(task_id, seed):
    first = reset(task_id, seed)
    first_state = oracle(task_id)
    second = reset(task_id, seed)
    assert first["instruction"] == second["instruction"]
    assert first_state == oracle(task_id)


@pytest.mark.parametrize("task_id", ALL_TASKS)
def test_different_seeds_differ(task_id):
    instructions = {reset(task_id, s)["instruction"] for s in range(20)}
    assert len(instructions) > 1


@pytest.mark.parametrize("task_id", ALL_TASKS)
def test_fresh_episode_is_not_solved(task_id):
    reset(task_id, seed=3)
    assert verify(task_id)["success"] is False


def test_task_page_renders():
    for task_id in EVERY_TASK:
        reset(task_id, seed=1)
        r = client.get(f"/task/{task_id}")
        assert r.status_code == 200
        assert "app" in r.text


def test_sessions_are_isolated():
    # two visitors on the same task must not share episode state
    client.post("/api/email-triage/reset?sid=alice", json={"seed": 1})
    client.post("/api/email-triage/reset?sid=bob", json={"seed": 1})
    alice_state = client.get("/api/email-triage/state?sid=alice").json()["state"]
    target = next(e for e in alice_state["emails"]
                  if e["sender"] == alice_state["target_sender"])
    client.post("/api/email-triage/action?sid=alice",
                json={"op": "archive", "id": target["id"]})
    bob_state = client.get("/api/email-triage/state?sid=bob").json()["state"]
    assert not any(e["archived"] for e in bob_state["emails"])
    # and the default (harness) session is untouched by either
    reset("email-triage", seed=1)
    assert verify("email-triage")["success"] is False


def test_landing_and_demo_pages():
    assert "WebTask Arena" in client.get("/").text
    assert client.get("/play").status_code == 200
    # /replays is 200 when assets are bundled, 404 otherwise — both acceptable
    assert client.get("/replays").status_code in (200, 404)


# --------------------------------------------------------------- email triage

def solve_email(task_id="email-triage"):
    state = oracle(task_id)
    for e in state["emails"]:
        if e["sender"] == state["target_sender"]:
            assert act(task_id, {"op": "archive", "id": e["id"]}).status_code == 200
    assert act(task_id, {"op": "star", "id": state["star_target_id"]}).status_code == 200


def test_email_triage_success():
    reset("email-triage", seed=42)
    solve_email()
    result = verify("email-triage")
    assert result["success"] is True, result


def test_email_triage_extra_archive_fails():
    reset("email-triage", seed=42)
    solve_email()
    state = oracle("email-triage")
    extra = next(e for e in state["emails"]
                 if e["sender"] != state["target_sender"] and not e["archived"])
    act("email-triage", {"op": "archive", "id": extra["id"]})
    result = verify("email-triage")
    assert result["success"] is False
    assert result["subgoals"]["no_extra_archived"] is False


def test_email_triage_wrong_star_fails():
    reset("email-triage", seed=42)
    state = oracle("email-triage")
    wrong = next(e for e in state["emails"] if e["id"] != state["star_target_id"])
    act("email-triage", {"op": "star", "id": wrong["id"]})
    assert verify("email-triage")["subgoals"]["starred_target_only"] is False


# ------------------------------------------------------------- settings panel

def test_settings_success():
    reset("settings-panel", seed=9)
    tz = oracle("settings-panel")["target_timezone"]
    r = act("settings-panel", {"op": "save", "draft": {"dark_mode": True, "timezone": tz}})
    assert r.status_code == 200
    assert verify("settings-panel")["success"] is True


def test_settings_unsaved_draft_does_not_count():
    # The trap this task exists for: toggling the UI without saving is failure.
    reset("settings-panel", seed=9)
    assert verify("settings-panel")["subgoals"]["dark_mode_saved"] is False


def test_settings_unrelated_change_fails():
    reset("settings-panel", seed=9)
    tz = oracle("settings-panel")["target_timezone"]
    act("settings-panel", {"op": "save",
                           "draft": {"dark_mode": True, "timezone": tz, "beta_features": True}})
    result = verify("settings-panel")
    assert result["success"] is False
    assert result["subgoals"]["no_unrelated_changes"] is False


# -------------------------------------------------------------- checkout form

def test_checkout_success():
    reset("checkout-form", seed=5)
    exp = oracle("checkout-form")["expected"]
    assert act("checkout-form", {"op": "contact", "name": exp["name"], "email": exp["email"]}).status_code == 200
    assert act("checkout-form", {"op": "shipping", "street": exp["street"],
                                 "city": exp["city"], "zip": exp["zip"]}).status_code == 200
    assert act("checkout-form", {"op": "submit"}).status_code == 200
    assert verify("checkout-form")["success"] is True


def test_checkout_validation_rejects_bad_input():
    reset("checkout-form", seed=5)
    r = act("checkout-form", {"op": "contact", "name": "", "email": "not-an-email"})
    assert r.status_code == 400
    assert set(r.json()["errors"]) == {"name", "email"}

    exp = oracle("checkout-form")["expected"]
    act("checkout-form", {"op": "contact", "name": exp["name"], "email": exp["email"]})
    r = act("checkout-form", {"op": "shipping", "street": exp["street"],
                              "city": exp["city"], "zip": "abc"})
    assert r.status_code == 400
    assert "zip" in r.json()["errors"]


def test_checkout_cannot_skip_steps():
    reset("checkout-form", seed=5)
    assert act("checkout-form", {"op": "submit"}).status_code == 400


def test_checkout_wrong_data_submits_but_fails_verify():
    # Submitting *something* is not success — fields must match the instruction.
    reset("checkout-form", seed=5)
    act("checkout-form", {"op": "contact", "name": "Wrong Person", "email": "wrong@example.com"})
    act("checkout-form", {"op": "shipping", "street": "1 Nowhere St", "city": "Nil", "zip": "00000"})
    act("checkout-form", {"op": "submit"})
    result = verify("checkout-form")
    assert result["subgoals"]["submitted"] is True
    assert result["success"] is False


# --------------------------------------------------------------- causal gate

def test_causal_gate_success():
    reset("causal-gate", seed=3)
    code = oracle("causal-gate")["expected_code"]
    act("causal-gate", {"op": "set_source", "value": "Friend referral"})
    act("causal-gate", {"op": "save", "draft": {"referral_code": code}})
    assert verify("causal-gate")["success"] is True


def test_causal_gate_code_without_source_fails():
    # Typing the right code into the right-shaped field isn't enough — the
    # source selection is the causal precondition, checked independently.
    reset("causal-gate", seed=3)
    code = oracle("causal-gate")["expected_code"]
    act("causal-gate", {"op": "save", "draft": {"referral_code": code}})
    result = verify("causal-gate")
    assert result["success"] is False
    assert result["subgoals"]["referral_code_correct"] is False


def test_causal_gate_only_final_save_counts():
    # Like settings-panel: draft state doesn't matter, only what's true at
    # the moment of the final save (flipping source back and forth is fine).
    reset("causal-gate", seed=3)
    code = oracle("causal-gate")["expected_code"]
    act("causal-gate", {"op": "set_source", "value": "Friend referral"})
    act("causal-gate", {"op": "set_source", "value": "Search engine"})
    act("causal-gate", {"op": "set_source", "value": "Friend referral"})
    act("causal-gate", {"op": "save", "draft": {"referral_code": code}})
    assert verify("causal-gate")["success"] is True


def test_causal_gate_decoy_field_fails():
    # The failure mode this task exists to catch: pattern-matching "code-shaped
    # field near a discount theme" instead of the actual causal precondition.
    reset("causal-gate", seed=3)
    code = oracle("causal-gate")["expected_code"]
    act("causal-gate", {"op": "save", "draft": {"gift_card": code}})
    result = verify("causal-gate")
    assert result["success"] is False
    assert result["subgoals"]["referral_code_correct"] is False
    assert result["subgoals"]["gift_card_untouched"] is False


# -------------------------------------------------------------- flash offer

def test_flash_offer_fresh_not_solved():
    reset("flash-offer", seed=2)
    assert verify("flash-offer")["success"] is False


def test_flash_offer_success_immediately_after_reset():
    # Well under CYCLE_SECONDS after reset, slot 0 is still the current code.
    reset("flash-offer", seed=1)
    codes = oracle("flash-offer")["codes"]
    r = act("flash-offer", {"op": "redeem", "code": codes[0]})
    assert r.status_code == 200
    assert verify("flash-offer")["success"] is True


def test_flash_offer_stale_code_fails():
    reset("flash-offer", seed=1)
    codes = oracle("flash-offer")["codes"]
    act("flash-offer", {"op": "redeem", "code": codes[1]})  # a different slot's code
    result = verify("flash-offer")
    assert result["success"] is False
    assert result["subgoals"]["code_matched_live_value"] is False


def test_flash_offer_codes_deterministic_per_seed():
    # The code *set* is seed-pure even though which one is "current" is not.
    reset("flash-offer", seed=1)
    codes_a = oracle("flash-offer")["codes"]
    reset("flash-offer", seed=1)
    codes_b = oracle("flash-offer")["codes"]
    assert codes_a == codes_b


def test_flash_offer_seeds_pick_different_code_sets():
    sets = set()
    for s in range(20):
        reset("flash-offer", seed=s)
        sets.add(tuple(oracle("flash-offer")["codes"]))
    assert len(sets) > 1