| """Preserved-model identity, actual probe protocol and resume boundaries (offline HTTP).""" |
| import copy |
| import json |
| from pathlib import Path |
|
|
| import httpx |
| import pytest |
| from common import read_json, write_json |
| from modeling import client |
| from modeling.architecture import validate_weights |
| from modeling.student import DEFAULT, configure |
| from modeling.teacher import configure as configure_teacher, validate as validate_teacher |
| from test_release_integration import ROOT, command |
|
|
|
|
| def test_teacher_root_custom_path_and_identity_are_sealed(config, tmp_path, monkeypatch): |
| monkeypatch.setenv('DECICO_RSI_MODELS_ROOT', '/stale/shell/Models') |
| assert configure_teacher(config)['teacher'] == config['teacher'] |
| selected = configure_teacher(config, models_root='/another/Models') |
| assert selected['teacher']['path'] == '/another/Models/Qwen3.8-27B' |
| selected = configure_teacher(config, model_path='/another/teacher-weights') |
| assert configure_teacher(selected)['teacher'] == selected['teacher'] |
| selected['data_service']['model'] = 'old-teacher' |
| with pytest.raises(ValueError, match='identity'): |
| validate_teacher(selected) |
| with pytest.raises(ValueError, match='absolute'): |
| configure_teacher(config, model_path='relative') |
|
|
|
|
| @pytest.mark.parametrize('name', ['Qwen3.5-35B-A3B', 'Qwen3.8-27B']) |
| def test_original_architectures_admitted_and_model_substitution_rejected(tmp_path, name): |
| from test_gdn_adapter import hybrid_config |
| path = tmp_path / name |
| path.mkdir() |
| actual = hybrid_config(name == 'Qwen3.5-35B-A3B') |
| write_json(path / 'config.json', actual) |
| assert validate_weights(path, name)['gdn'] is True |
| write_json(path / 'config.json', {**actual, 'model_type': 'substituted-model'}) |
| with pytest.raises(ValueError, match='differs'): |
| validate_weights(path, name) |
| with pytest.raises(ValueError, match='Unsupported'): |
| validate_weights(path, 'Qwen3-32B') |
|
|
|
|
| def test_teacher_override_survives_suite_prepare_and_changed_teacher_cannot_resume(tmp_path): |
| suite = tmp_path / 'suite' |
| args = [ROOT / 'run_experiments.py', '--output', suite, '--seeds', 811, 812, |
| '--student-model', 'Qwen3.5-35B-A3B', '--models-root', '/external/Models', |
| '--teacher-model-path', '/external/renamed-teacher', '--prepare-only'] |
| result = command(*args) |
| assert result.returncode == 0, result.stdout + result.stderr |
| for seed in (811, 812): |
| cfg = read_json(suite / f'seed-{seed}/config.json') |
| assert cfg['student']['path'] == '/external/Models/Qwen3.5-35B-A3B' |
| assert cfg['teacher'] == {'name': 'Qwen3.8-27B', 'path': '/external/renamed-teacher'} |
| changed = list(args) |
| changed[changed.index('--teacher-model-path') + 1] = '/external/other-teacher' |
| result = command(*changed, '--resume') |
| assert result.returncode != 0 and 'Suite model' in result.stderr |
|
|
|
|
| def mock_services(monkeypatch, config, failure=None): |
| """GET always passes; generation may fail independently, as on the server.""" |
| seen = [] |
| config['serving_backend']['receipt_dirs'] = {'student': '/offline/student', 'teacher': '/offline/teacher'} |
| def transport(request): |
| is_teacher = request.url.port == 5058 |
| role = 'teacher' if is_teacher else 'student' |
| expected = config['serving_requirements'][role] |
| if request.method == 'GET': |
| return httpx.Response(200, json={'data': [ |
| {'id': expected['model'], 'max_model_len': expected['max_model_len']}]}) |
| body = json.loads(request.content) |
| seen.append((role, body)) |
| assert body['model'] == expected['model'] |
| assert body['chat_template_kwargs'] == {'enable_thinking': False} |
| assert not any(k in body for k in ('max_tokens', 'max_completion_tokens')) |
| if failure == 'http500' or (failure == 'teacher500' and is_teacher): |
| return httpx.Response(500, json={'error': 'Ninja build failed'}) |
| if failure == 'transport': |
| raise httpx.ConnectError('Connection refused', request=request) |
| content = json.dumps({'ready': True}) if is_teacher else json.dumps([ |
| {'name': 'service_probe_echo', 'arguments': {'text': 'hello'}}]) |
| if failure == 'malformed': |
| content = 'not-json' |
| finish = 'length' if failure == 'length' else 'stop' |
| return httpx.Response(200, json={'model': expected['model'], 'choices': [ |
| {'message': {'role': 'assistant', 'content': content}, 'finish_reason': finish}], |
| 'usage': {'prompt_tokens': 10, 'completion_tokens': 5}}) |
| student = httpx.Client(base_url='http://127.0.0.1:5057/v1', transport=httpx.MockTransport(transport)) |
| teacher = httpx.Client(base_url='http://127.0.0.1:5058/v1', transport=httpx.MockTransport(transport)) |
| monkeypatch.setattr(client, '_http', lambda: student) |
| monkeypatch.setattr(client, '_data_http', lambda: teacher) |
| monkeypatch.setattr(client, 'BASE_MODEL_NAME', config['student']['name']) |
| monkeypatch.setattr(client, 'DATA_MODEL', config['teacher']['name']) |
| return seen |
|
|
|
|
| def test_live_probe_uses_real_clients_and_handwritten_inputs_only(monkeypatch, tmp_path, config): |
| from controller.call_meter import CallMeter, activate |
| from modeling.service_probe import run |
| seen = mock_services(monkeypatch, config) |
| monkeypatch.setattr('modeling.backend_receipts.verify', lambda cfg: {'fixture': True}) |
| target = tmp_path / 'live.json' |
| with activate(CallMeter(tmp_path / 'calls.json', None)): |
| report = run(config, target) |
| assert report['status'] == 'passed' and len(seen) == 4 |
| assert report['benchmark_rows_read'] == 0 |
| assert report['used_for_learning_or_selection'] is False |
| assert report['training_or_lora_tested'] is False |
| calls = read_json(tmp_path / 'calls.json')['calls'].values() |
| assert len(calls) == 4 and all(c['role'] == 'service_preflight' for c in calls) |
| assert all(c['status'] == 'complete' for c in calls) |
| assert read_json(target) == report |
| assert {b['model'] for _, b in seen} == {DEFAULT, 'Qwen3.8-27B'} |
| assert 'service_probe_echo' in seen[0][1]['messages'][0]['content'] |
|
|
|
|
| @pytest.mark.parametrize('failure', ['http500', 'teacher500', 'transport', 'malformed', 'length']) |
| def test_models_listing_alone_cannot_pass_live_preflight(monkeypatch, tmp_path, config, failure): |
| from modeling.service_probe import run |
| mock_services(monkeypatch, config, failure) |
| target = tmp_path / 'failed.json' |
| with pytest.raises(client.ServiceIdentityError, match='preflight failed'): |
| run(config, target) |
| report = read_json(target) |
| assert report['status'] == 'failed' |
| assert report['failed_role'] == ('teacher' if failure == 'teacher500' else 'student') |
|
|
|
|
| def test_teacher_failure_blocks_baseline_without_changing_learning_state(monkeypatch, tmp_path, config): |
| import rsi_loop |
| from modeling import service_probe |
| monkeypatch.setattr(client, 'verify_serving_profile', lambda *args: {'fixture': True}) |
| def fail(*args): |
| raise client.ServiceIdentityError('Teacher forward returned 500') |
| monkeypatch.setattr(service_probe, 'check_generation', fail) |
| target = tmp_path / 'run' |
| with pytest.raises(client.ServiceIdentityError, match='Teacher'): |
| rsi_loop._run(config, target) |
| assert read_json(target / 'status.json')['phase'] == 'service_preflight' |
| assert not (target / 'baseline.json').exists() |
| assert not (target / 'iterations').exists() |
| assert not (target / 'history.json').exists() |
|
|
