File size: 8,166 Bytes
115b94d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
"""OpenAI wire format, role isolation and ledger protections; no live calls."""
import json
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
import httpx
from src.adapters.llm.openai_code import OpenAICodeProvider,MODELS,ENDPOINT,read_token,providers_from_connection
from src.modules.llm_gateway.routing import VERSION,plan,catalog
from src.modules.llm_gateway.service import LLMGateway
from src.modules.skill.domain.models import SkillError
from src.tests.runtime.fixtures import setup,workspace,USER,SOURCE


class OpenAICodeTests(unittest.TestCase):
    def provider(self,handler=None,model=MODELS[0]):
        p=OpenAICodeProvider(model=model,token='sk-synthetic-openai-test-only',transport=httpx.MockTransport(handler) if handler else None)
        p.health={'status':'healthy'}
        return p

    def test_medium_wire_format_counts_all_completion_tokens(self):
        def reply(req):
            body=json.loads(req.content)
            self.assertEqual(str(req.url),ENDPOINT)
            self.assertEqual(body['reasoning_effort'],'medium')
            self.assertEqual(body['max_completion_tokens'],16384)
            self.assertFalse(body['store'])
            self.assertNotIn('max_tokens',body);self.assertNotIn('enable_thinking',body)
            self.assertNotIn('temperature',body);self.assertNotIn('tools',body)
            return httpx.Response(200,json={'id':'chatcmpl-openai-test',
                'choices':[{'finish_reason':'stop','message':{'content':json.dumps({'files':SOURCE})}}],
                'usage':{'prompt_tokens':100,'completion_tokens':80,'completion_tokens_details':{'reasoning_tokens':60}}})
        with workspace() as folder:
            rt,request,_,_=setup(folder);r=rt.create(USER,request,'openai-ledger-test')
            rt.store.mutate(r['id'],lambda r:r.update(status='running'))
            body=LLMGateway(rt.store,[self.provider(reply)]).generate(r['id'],'code',{},lambda:False)
            self.assertEqual(body['files'],SOURCE)
            call=rt.store.calls(r['id'])[0]
            self.assertEqual(call['charged'],180) # Reasoning tokens already included in 80.
            self.assertEqual(call['metadata']['reasoning_effort'],'medium')
            self.assertNotIn('sk-synthetic-openai-test-only',json.dumps(call))

    def test_priority_is_61_astra_55_in_all_requested_roles_and_picker(self):
        legacy=SimpleNamespace(name='legacy',model='kimi-k3',routing_version=VERSION,selectable=True,
            task_roles=['chat','design','diagnose','code','repair'],health={'status':'healthy'})
        providers=[legacy,*[self.provider(model=m) for m in reversed(MODELS)]]
        for stage in ('chat','design','diagnose','code','repair'):
            self.assertEqual([p.model for p in plan(providers,stage,{})[0]],list(MODELS)+['kimi-k3'])
        self.assertEqual(catalog(providers)['effective_policy']['java-code'],MODELS[0])
        self.assertEqual(catalog(providers)['effective_policy']['design'],MODELS[0])
        self.assertEqual(catalog(providers)['output_limits']['design'],8192)
        self.assertEqual(catalog(providers)['output_limits']['code'],16384)
        self.assertTrue(all(m['selectable'] for m in catalog(providers)['models']))
        for model in MODELS:
            self.assertEqual(plan(providers,'design',{'request':{'selected_model':model}})[0][0].model,model)

    def test_unhealthy_model_is_excluded_and_manual_choice_is_preserved(self):
        p=self.provider();p.health={'status':'unknown'}
        self.assertEqual(plan([p],'code',{})[0],[])
        legacy=SimpleNamespace(name='legacy',model='kimi-k3',routing_version=VERSION,selectable=True,
            task_roles=['code'],health={'status':'healthy'})
        self.assertEqual(plan([self.provider(),legacy],'code',{'request':{'selected_model':'kimi-k3'}})[0],[legacy])

    def test_design_medium_has_reasoning_space_and_retry_respects_unlimited_policy(self):
        limits=[]
        def reply(req):
            body=json.loads(req.content);limits.append(body['max_completion_tokens'])
            self.assertEqual(body['reasoning_effort'],'medium')
            return httpx.Response(200,json={'id':'chatcmpl-design-test',
                'choices':[{'finish_reason':'length' if len(limits)==1 else 'stop',
                            'message':{'content':'{"summary":"concise design"}'}}],
                'usage':{'prompt_tokens':100,'completion_tokens':8192 if len(limits)==1 else 100}})
        with workspace() as folder:
            rt,request,_,_=setup(folder)
            request['budget_policy']={'mode':'unlimited-usage-v1'}
            run=rt.create(USER,request,'openai-design-space')
            rt.store.mutate(run['id'],lambda r:r.update(status='running'))
            result=LLMGateway(rt.store,[self.provider(reply)]).generate(run['id'],'design',{},lambda:False,token_limit=1000)
            self.assertEqual(result,{'summary':'concise design'})
            self.assertEqual(limits,[8192,16384])
            self.assertTrue(all(c['state']=='settled' for c in rt.store.calls(run['id'])))

    def test_unknown_transport_keeps_reservation_and_never_tries_backup(self):
        seen=[]
        def timeout(req):seen.append('first');raise httpx.ReadTimeout('synthetic')
        def backup(req):seen.append('backup');raise AssertionError('must not replay')
        with workspace() as folder:
            rt,request,_,_=setup(folder);r=rt.create(USER,request,'openai-unknown-test')
            rt.store.mutate(r['id'],lambda r:r.update(status='running'))
            gateway=LLMGateway(rt.store,[self.provider(timeout),self.provider(backup,MODELS[1])])
            with self.assertRaises(SkillError):gateway.generate(r['id'],'code',{},lambda:False)
            self.assertEqual(seen,['first'])
            call=rt.store.calls(r['id'])[0]
            self.assertEqual(call['state'],'unknown');self.assertIsNone(call['charged'])

    def test_connection_requires_fresh_matching_credential_validation(self):
        with workspace() as folder:
            root=Path(folder);key=root/'key.txt';key.write_text('sk-synthetic-openai-test-only',encoding='utf8')
            config={'endpoint':ENDPOINT,'credential_source':str(key),'reasoning_effort':'medium'}
            health={**config,'credential_mtime_ns':key.stat().st_mtime_ns,'checked_at':time.time(),
                'models':{MODELS[0]:{'status':'healthy'},MODELS[1]:{'status':'not_probed'}}}
            path=root/'connection.json';path.write_text(json.dumps(config),encoding='utf8')
            h=root/'health.json';h.write_text(json.dumps(health),encoding='utf8')
            self.assertEqual([p.model for p in providers_from_connection(path)],[MODELS[0]])
            health['checked_at']-=86401;h.write_text(json.dumps(health),encoding='utf8')
            self.assertEqual(providers_from_connection(path),[])
            health['checked_at']=time.time();health['credential_mtime_ns']+=1
            h.write_text(json.dumps(health),encoding='utf8')
            self.assertEqual(providers_from_connection(path),[])
            config['endpoint']='https://unapproved.example/v1/chat/completions'
            path.write_text(json.dumps(config),encoding='utf8')
            with self.assertRaises(ValueError):providers_from_connection(path)

    def test_token_reader_rejects_ambiguous_keys_without_echo(self):
        with workspace() as folder:
            p=Path(folder)/'key.txt';p.write_text('sk-first-secret sk-second-secret',encoding='utf8')
            with self.assertRaisesRegex(ValueError,'^OPENAI_CREDENTIAL_CONFIGURATION_INVALID$'):read_token(p)
        with self.assertRaises(ValueError):OpenAICodeProvider(model='gpt-unapproved',token='sk-synthetic')

    def test_duplicate_listener_is_rejected_before_workbench_initialization(self):
        import socket
        from scripts.serve_local import require_available_ports
        with socket.socket() as listener:
            listener.bind(('127.0.0.1',0));listener.listen()
            port=listener.getsockname()[1]
            with self.assertRaisesRegex(RuntimeError,'LOCAL_LISTENER_ALREADY_IN_USE'):
                require_available_ports(port)