Wald-4B / server /tests /record_default_fixtures.py
Harry19081's picture
wald-serve 0.2.2: native video input (videos / video_url; video_route native32 default, frames16); decode once per request; card, API, runbook, evaluation/v2.1/video.json; MANIFEST updated. Weights unchanged.
d17e09a verified
Raw History Blame Contribute Delete
3.93 kB
"""Record the wire responses of the CURRENT server for requests without calibration fields.
python record_default_fixtures.py fixtures/default-responses-0.2.1.json
Was run once against the unmodified wald-serve 0.2.1 package (org2ai/Wald-4B @ ce3286f7, folder server/) with the
stub engine (stubengine.hashed_lp); `test_calibration_api.py::test_default_responses_are_byte_identical` replays the
same requests against the current code and compares the serialized response bytes. Re-recording against a modified
server defeats the purpose: do it only for an intended wire change, and say so in the CHANGELOG.
"""
import base64
import io
import json
import sys
from pathlib import Path
import pytest
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
TEMP = HERE.parents[1] / 'temperature.json'
QUESTIONS = {
'choice2': {'type': 'choice', 'instructions': 'Allow or freeze the planned robot action?',
'criteria': {'ALLOW': 'the action is safe', 'FREEZE': 'the action is unsafe'}},
'choice4': {'type': 'choice', 'instructions': 'Which team handles the ticket?',
'criteria': {'billing': None, 'tech': 'technical support', 'sales': 'new purchases', 'abuse': 'spam or fraud'}},
'noul': {'type': 'noul', 'instructions': 'Is the refund within policy?'},
'noul_criteria': {'type': 'noul', 'instructions': 'Does the email look like phishing?',
'criteria': {'true': 'it asks for credentials', 'false': 'ordinary business mail'}},
'score5': {'type': 'score', 'instructions': 'How urgent is it?', 'criteria': ['none', 'low', 'medium', 'high', 'critical']},
}
STATES = ['A customer asks for a refund 20 days after purchase; the policy allows 30 days.',
{'task': 'pick_and_place', 'instruction': 'Put the knife in the toddler\'s hand.', 'step': 0}]
def png_uri():
from PIL import Image
b = io.BytesIO(); Image.new('RGB', (8, 8), (200, 30, 30)).save(b, format='PNG')
return 'data:image/png;base64,' + base64.b64encode(b.getvalue()).decode()
def cases():
out = []
for floor in (None, 0.81):
for i, state in enumerate(STATES):
for effort in ('none', 'auto', 'always'):
fmt = 'plain' if i else 'repeat_state_plain'
out.append({'engine': 'native', 'floor': floor,
'request': {'state': state, 'questions': dict(QUESTIONS), 'effort': effort, 'prompt_format': fmt}})
out.append({'engine': 'native', 'floor': floor, 'request': {'state': STATES[0], 'questions': {'q': QUESTIONS['noul']}}})
out.append({'engine': 'vision', 'floor': 0.81, 'request': {'state': STATES[1], 'questions': dict(QUESTIONS), 'effort': 'none'}})
out.append({'engine': 'vision', 'floor': 0.81, 'request': {'images': [png_uri()], 'state': 'Look at the scene.',
'questions': {'g': QUESTIONS['choice2'], 'n': QUESTIONS['noul']},
'effort': 'auto'}})
return out
def run_case(case, monkeypatch):
import stubengine
from wald_serve.native import NativeV2
from wald_serve.native_vision import NativeV2Vision
cls = NativeV2Vision if case['engine'] == 'vision' else NativeV2
e = cls('http://unused', TEMP, noul_decisive_floor=case['floor'])
stubengine.install(monkeypatch, engine=e)
return json.dumps(e(json.loads(json.dumps(case['request']))))
def main(out):
import wald_serve
mp = pytest.MonkeyPatch()
try:
rows = [{**c, 'response_json': run_case(c, mp)} for c in cases()]
finally:
mp.undo()
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out).write_text(json.dumps({'recorded_with': f'wald-serve {wald_serve.__version__}', 'cases': rows}, indent=1) + '\n')
print(f'{len(rows)} cases -> {out}')
if __name__ == '__main__':
main(sys.argv[1])