VibeCoding / src /tests /runtime /test_model_selection.py
dellweb3's picture
Deploy verified conversational workbench and isolated Java application previews
5ca3c16 verified
Raw History Blame Contribute Delete
3.24 kB
"""A selected API Code must be used exactly, without silent fallback."""
import time
import unittest
from types import SimpleNamespace
from pathlib import Path
from src.adapters.persistence.local_identity import LocalIdentity
from src.apps.workbench.conversation import Conversations
from src.modules.llm_gateway.routing import VERSION,plan,selected_provider
from src.modules.llm_gateway.service import ModelReply
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import workspace
class ModelSelectionTests(unittest.TestCase):
def provider(self,code,seen):
return SimpleNamespace(model=code,name=code,routing_version=VERSION,selectable=True,
task_roles=['chat','design','diagnose','code','repair'],health={'status':'healthy'},
complete=lambda messages,limit,cancelled:seen.append(code) or ModelReply({'answer':'ok'},20,5,code))
def test_task_plan_pins_exact_code_and_never_falls_back(self):
seen=[];providers=[self.provider('qwen3.8-flash',seen),self.provider('kimi-k3',seen)]
run={'request':{'selected_model':'kimi-k3'},'project_id':'webapp-demo','snapshot':{'framework':{'dependencies':{}}}}
chosen,reason=plan(providers,'code',run)
self.assertEqual([p.model for p in chosen],['kimi-k3'])
self.assertEqual(reason,'user-selected')
providers[1].health={'status':'rejected'}
self.assertEqual(plan(providers,'repair',run)[0],[])
with self.assertRaises(SkillError):selected_provider(providers,'unknown-code',['chat'])
def test_automatic_plan_includes_available_selectable_alternatives(self):
seen=[]
providers=[self.provider(code,seen) for code in ['qwen3.8-flash','kimi-k3','deepseek-v4-flash-0731','qwen3.8-max-0902','deepseek-v4.1-flash-1017']]
run={'request':{},'project_id':'erp-demo','snapshot':{'framework':{'dependencies':{}}}}
for role in ['chat','design','code','repair']:
candidates,_=plan(providers,role,run)
self.assertEqual({p.model for p in candidates},{p.model for p in providers})
providers[1].health={'status':'rejected'}
self.assertNotIn('kimi-k3',[p.model for p in plan(providers,'code',run)[0]])
def test_chat_dispatches_selected_code_and_records_it(self):
identity=LocalIdentity(Path(self.enterContext(workspace()))/'identity.sqlite')
identity.seed_projects();identity.seed_demo_users()
token,_=identity.login('programmer','VibeCoding@2026');actor=identity.session(token)[0]
seen=[];service=Conversations(identity,[self.provider('qwen3.8-flash',seen),self.provider('kimi-k3',seen)])
row=service.submit(actor,'conversation','hello','chosen-model-key',model='kimi-k3')
for _ in range(100):
result=service.get(actor,row['id'])
if result['state']!='running':break
time.sleep(.01)
self.assertEqual(result['state'],'completed')
self.assertEqual(result['model'],'kimi-k3')
self.assertEqual(seen,['kimi-k3'])
with self.assertRaises(SkillError):
service.submit(actor,'conversation','hello','chosen-model-key',model='qwen3.8-flash')
if __name__=='__main__':unittest.main()