"""Preserved-model identity, actual probe protocol and resume boundaries (offline HTTP).""" import copy import json from pathlib import Path import httpx import pytest from common import read_json, write_json from modeling import client from modeling.architecture import validate_weights from modeling.student import DEFAULT, configure from modeling.teacher import configure as configure_teacher, validate as validate_teacher from test_release_integration import ROOT, command def test_teacher_root_custom_path_and_identity_are_sealed(config, tmp_path, monkeypatch): monkeypatch.setenv('DECICO_RSI_MODELS_ROOT', '/stale/shell/Models') assert configure_teacher(config)['teacher'] == config['teacher'] selected = configure_teacher(config, models_root='/another/Models') assert selected['teacher']['path'] == '/another/Models/Qwen3.8-27B' selected = configure_teacher(config, model_path='/another/teacher-weights') assert configure_teacher(selected)['teacher'] == selected['teacher'] selected['data_service']['model'] = 'old-teacher' with pytest.raises(ValueError, match='identity'): validate_teacher(selected) with pytest.raises(ValueError, match='absolute'): configure_teacher(config, model_path='relative') @pytest.mark.parametrize('name', ['Qwen3.5-35B-A3B', 'Qwen3.8-27B']) def test_original_architectures_admitted_and_model_substitution_rejected(tmp_path, name): from test_gdn_adapter import hybrid_config path = tmp_path / name path.mkdir() actual = hybrid_config(name == 'Qwen3.5-35B-A3B') write_json(path / 'config.json', actual) assert validate_weights(path, name)['gdn'] is True write_json(path / 'config.json', {**actual, 'model_type': 'substituted-model'}) with pytest.raises(ValueError, match='differs'): validate_weights(path, name) with pytest.raises(ValueError, match='Unsupported'): validate_weights(path, 'Qwen3-32B') def test_teacher_override_survives_suite_prepare_and_changed_teacher_cannot_resume(tmp_path): suite = tmp_path / 'suite' args = [ROOT / 'run_experiments.py', '--output', suite, '--seeds', 811, 812, '--student-model', 'Qwen3.5-35B-A3B', '--models-root', '/external/Models', '--teacher-model-path', '/external/renamed-teacher', '--prepare-only'] result = command(*args) assert result.returncode == 0, result.stdout + result.stderr for seed in (811, 812): cfg = read_json(suite / f'seed-{seed}/config.json') assert cfg['student']['path'] == '/external/Models/Qwen3.5-35B-A3B' assert cfg['teacher'] == {'name': 'Qwen3.8-27B', 'path': '/external/renamed-teacher'} changed = list(args) changed[changed.index('--teacher-model-path') + 1] = '/external/other-teacher' result = command(*changed, '--resume') assert result.returncode != 0 and 'Suite model' in result.stderr def mock_services(monkeypatch, config, failure=None): """GET always passes; generation may fail independently, as on the server.""" seen = [] config['serving_backend']['receipt_dirs'] = {'student': '/offline/student', 'teacher': '/offline/teacher'} def transport(request): is_teacher = request.url.port == 5058 role = 'teacher' if is_teacher else 'student' expected = config['serving_requirements'][role] if request.method == 'GET': return httpx.Response(200, json={'data': [ {'id': expected['model'], 'max_model_len': expected['max_model_len']}]}) body = json.loads(request.content) seen.append((role, body)) assert body['model'] == expected['model'] assert body['chat_template_kwargs'] == {'enable_thinking': False} assert not any(k in body for k in ('max_tokens', 'max_completion_tokens')) if failure == 'http500' or (failure == 'teacher500' and is_teacher): return httpx.Response(500, json={'error': 'Ninja build failed'}) if failure == 'transport': raise httpx.ConnectError('Connection refused', request=request) content = json.dumps({'ready': True}) if is_teacher else json.dumps([ {'name': 'service_probe_echo', 'arguments': {'text': 'hello'}}]) if failure == 'malformed': content = 'not-json' finish = 'length' if failure == 'length' else 'stop' return httpx.Response(200, json={'model': expected['model'], 'choices': [ {'message': {'role': 'assistant', 'content': content}, 'finish_reason': finish}], 'usage': {'prompt_tokens': 10, 'completion_tokens': 5}}) student = httpx.Client(base_url='http://127.0.0.1:5057/v1', transport=httpx.MockTransport(transport)) teacher = httpx.Client(base_url='http://127.0.0.1:5058/v1', transport=httpx.MockTransport(transport)) monkeypatch.setattr(client, '_http', lambda: student) monkeypatch.setattr(client, '_data_http', lambda: teacher) monkeypatch.setattr(client, 'BASE_MODEL_NAME', config['student']['name']) monkeypatch.setattr(client, 'DATA_MODEL', config['teacher']['name']) return seen def test_live_probe_uses_real_clients_and_handwritten_inputs_only(monkeypatch, tmp_path, config): from controller.call_meter import CallMeter, activate from modeling.service_probe import run seen = mock_services(monkeypatch, config) monkeypatch.setattr('modeling.backend_receipts.verify', lambda cfg: {'fixture': True}) target = tmp_path / 'live.json' with activate(CallMeter(tmp_path / 'calls.json', None)): report = run(config, target) assert report['status'] == 'passed' and len(seen) == 4 assert report['benchmark_rows_read'] == 0 assert report['used_for_learning_or_selection'] is False assert report['training_or_lora_tested'] is False calls = read_json(tmp_path / 'calls.json')['calls'].values() assert len(calls) == 4 and all(c['role'] == 'service_preflight' for c in calls) assert all(c['status'] == 'complete' for c in calls) assert read_json(target) == report assert {b['model'] for _, b in seen} == {DEFAULT, 'Qwen3.8-27B'} assert 'service_probe_echo' in seen[0][1]['messages'][0]['content'] @pytest.mark.parametrize('failure', ['http500', 'teacher500', 'transport', 'malformed', 'length']) def test_models_listing_alone_cannot_pass_live_preflight(monkeypatch, tmp_path, config, failure): from modeling.service_probe import run mock_services(monkeypatch, config, failure) target = tmp_path / 'failed.json' with pytest.raises(client.ServiceIdentityError, match='preflight failed'): run(config, target) report = read_json(target) assert report['status'] == 'failed' assert report['failed_role'] == ('teacher' if failure == 'teacher500' else 'student') def test_teacher_failure_blocks_baseline_without_changing_learning_state(monkeypatch, tmp_path, config): import rsi_loop from modeling import service_probe monkeypatch.setattr(client, 'verify_serving_profile', lambda *args: {'fixture': True}) def fail(*args): raise client.ServiceIdentityError('Teacher forward returned 500') monkeypatch.setattr(service_probe, 'check_generation', fail) target = tmp_path / 'run' with pytest.raises(client.ServiceIdentityError, match='Teacher'): rsi_loop._run(config, target) assert read_json(target / 'status.json')['phase'] == 'service_preflight' assert not (target / 'baseline.json').exists() assert not (target / 'iterations').exists() assert not (target / 'history.json').exists()