File size: 4,225 Bytes
63d4e3a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import json

from tests.conftest import VALID_OUTPUT


GENERATE = "/v1/progression/generate"

OLD_METADATA_FIELDS = {
    "metadata",
    "model_repo",
    "model_file",
    "version",
    "quantization",
    "runtime",
    "elapsed_seconds",
}


def test_health_ready(client):
    assert client.get("/health").json() == {
        "status": "ok",
        "model_ready": True,
    }


def test_health_degraded(client, fake_runtime):
    fake_runtime.is_ready = False

    assert client.get("/health").json() == {
        "status": "degraded",
        "model_ready": False,
    }


def test_models(client):
    entry = client.get("/models").json()["progression"]["v1"]

    assert entry["runtime"] == "llama-cpp-python"
    assert entry["quantization"] == "Q8_0"
    assert entry["prompt_format"] == "raw_sft_completion"
    assert entry["status"] == "ready"


def test_generate_success(client, fake_runtime, valid_request):
    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 200

    body = response.json()

    assert set(body) == {"progression", "usage"}
    assert body["progression"] == VALID_OUTPUT["progression"]
    assert set(body["usage"]) == {"input_tokens", "output_tokens"}
    assert body["usage"]["input_tokens"] > 0
    assert body["usage"]["output_tokens"] > 0
    assert not OLD_METADATA_FIELDS & set(body)

    # The exact raw SFT prompt is what reaches the model.
    assert len(fake_runtime.prompts) == 1
    assert fake_runtime.prompts[0].endswith("\n\nOUTPUT:\n")
    assert (
        '"previous_activity":{"title":"Practice the skill"}'
        in fake_runtime.prompts[0]
    )


def test_generation_field_rejected(client, fake_runtime, valid_request):
    valid_request["generation"] = {"temperature": 1.0, "seed": 7}

    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 422
    assert fake_runtime.prompts == []


def test_public_schema_has_no_generation_field(client):
    schema = client.get("/openapi.json").json()
    properties = schema["components"]["schemas"]["ProgressionRequest"][
        "properties"
    ]

    assert set(properties) == {
        "version",
        "knowledge_submission",
        "wisdom",
        "village_goal",
        "previous_activity_title",
    }


def test_old_wisdom_schema_rejected(client, valid_request):
    valid_request["wisdom"] = {
        "title": "Past learning",
        "description": "Older record",
        "content": "I learned something before.",
    }

    assert client.post(GENERATE, json=valid_request).status_code == 422


def test_wisdom_capitalization_enforced(client, valid_request):
    wisdom = valid_request["wisdom"]
    wisdom["bookType"] = wisdom.pop("BookType")

    assert client.post(GENERATE, json=valid_request).status_code == 422


def test_unknown_version(client, fake_runtime, valid_request):
    valid_request["version"] = "v2"

    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 404
    assert "v2" in response.json()["detail"]
    assert fake_runtime.prompts == []


def test_model_not_ready(client, fake_runtime, valid_request):
    fake_runtime.is_ready = False

    assert client.post(GENERATE, json=valid_request).status_code == 503


def test_text_before_json_is_accepted(client, fake_runtime, valid_request):
    fake_runtime.output = "Sure:\n" + json.dumps(VALID_OUTPUT)

    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 200
    assert response.json()["progression"] == VALID_OUTPUT["progression"]


def test_malformed_json_returns_502(client, fake_runtime, valid_request):
    fake_runtime.output = '{"progression": {"knowledge_state": "COMP'

    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 502
    assert response.json()["detail"]["raw_output"] == fake_runtime.output


def test_invalid_schema_returns_502(client, fake_runtime, valid_request):
    fake_runtime.output = json.dumps(
        {"progression": {"knowledge_state": "DONE"}}
    )

    response = client.post(GENERATE, json=valid_request)

    assert response.status_code == 502
    assert "validation_error" in response.json()["detail"]