File size: 10,767 Bytes
caf8059
 
 
 
5ca3c16
 
caf8059
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5ca3c16
 
caf8059
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115b94d
caf8059
 
 
 
 
115b94d
 
 
 
 
 
 
 
caf8059
 
 
 
 
 
115b94d
caf8059
 
 
 
 
 
 
 
115b94d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
caf8059
5ca3c16
caf8059
 
 
 
 
 
 
5ca3c16
caf8059
 
 
 
 
 
 
 
 
 
 
5ca3c16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
caf8059
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
"""Atomic local run state, fenced claims and durable model reservations."""
import json
import sqlite3
import time
import re
import traceback
from contextlib import contextmanager
from pathlib import Path
from src.modules.skill.domain.models import SkillError, canonical, digest


class RuntimeStore:
    def __init__(self, path):
        self.path = Path(path)
        self.path.parent.mkdir(parents=True, exist_ok=True)
        with self.connect() as db:
            db.executescript('''
            CREATE TABLE IF NOT EXISTS runs(id TEXT PRIMARY KEY, tenant TEXT NOT NULL, owner TEXT NOT NULL,
              request_key TEXT NOT NULL, request_digest TEXT NOT NULL, body TEXT NOT NULL,
              UNIQUE(tenant,owner,request_key));
            CREATE TABLE IF NOT EXISTS calls(id INTEGER PRIMARY KEY, run_id TEXT NOT NULL, reservation INTEGER NOT NULL,
              charged INTEGER, state TEXT NOT NULL, provider TEXT NOT NULL, prompt_digest TEXT NOT NULL);
            CREATE TABLE IF NOT EXISTS runtime_events(id INTEGER PRIMARY KEY, run_id TEXT NOT NULL,
              timestamp REAL NOT NULL, action TEXT NOT NULL, body_digest TEXT NOT NULL);
            CREATE TABLE IF NOT EXISTS runtime_stage_events(id INTEGER PRIMARY KEY,run_id TEXT NOT NULL,
              timestamp REAL NOT NULL,stage TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL);
            CREATE TABLE IF NOT EXISTS runtime_diagnostics(id INTEGER PRIMARY KEY,run_id TEXT NOT NULL,
              timestamp REAL NOT NULL,component TEXT NOT NULL,code TEXT NOT NULL,exception_type TEXT NOT NULL,frames TEXT NOT NULL);
            ''')
            if 'metadata' not in {r['name'] for r in db.execute('PRAGMA table_info(calls)')}:
                db.execute("ALTER TABLE calls ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'")

    @contextmanager
    def connect(self):
        db = sqlite3.connect(self.path, timeout=30)
        db.row_factory = sqlite3.Row
        try:
            db.execute('BEGIN IMMEDIATE')
            yield db
            db.commit()
        except BaseException:
            db.rollback()
            raise
        finally:
            db.close()

    def create(self, run, key, request):
        with self.connect() as db:
            old = db.execute('SELECT * FROM runs WHERE tenant=? AND owner=? AND request_key=?',
                             (run['tenant_id'], run['owner_id'], key)).fetchone()
            if old:
                if old['request_digest'] != digest(request):
                    raise SkillError('IDEMPOTENCY_CONFLICT', '同一请求键输入发生变化。', 409)
                return json.loads(old['body'])
            db.execute('INSERT INTO runs VALUES(?,?,?,?,?,?)', (run['id'], run['tenant_id'], run['owner_id'], key, digest(request), canonical(run)))
            return run

    def get(self, run_id):
        with self.connect() as db:
            row = db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()
            if not row:
                raise SkillError('RUN_NOT_FOUND', '任务不存在或不可见。', 404)
            return json.loads(row['body'])

    def mutate(self, run_id, fn, *, with_calls=False):
        with self.connect() as db:
            row = db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()
            if not row:
                raise SkillError('RUN_NOT_FOUND', '任务不存在或不可见。', 404)
            body = json.loads(row['body'])
            if with_calls:
                # Approval validates the ledger under the same write transaction.
                # Idempotent decisions must not change the task version.
                before=canonical(body)
                calls=[{**dict(r),'metadata':json.loads(r['metadata'])} for r in db.execute('SELECT * FROM calls WHERE run_id=? ORDER BY id',(run_id,))]
                fn(body,calls)
                if canonical(body)==before:return body
            else:fn(body)
            body['revision'] += 1
            db.execute('UPDATE runs SET body=? WHERE id=?', (canonical(body), run_id))
            db.execute('INSERT INTO runtime_events(run_id,timestamp,action,body_digest) VALUES(?,?,?,?)',
                       (run_id, time.time(), body['status'], digest(body)))
            return body

    def reserve(self, run_id, provider, prompt, amount, metadata=None, *, overdraft=None):
        with self.connect() as db:
            run = json.loads(db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()['body'])
            if run['status'] != 'running' or run.get('cancel_requested'):
                raise SkillError('RUN_NOT_ACTIVE', '任务不允许调用模型。', 409)
            rows = db.execute('SELECT * FROM calls WHERE run_id=?', (run_id,)).fetchall()
            used = sum(r['charged'] if r['charged'] is not None else r['reservation'] for r in rows)
            if any(r['state'] in {'reserved', 'unknown'} for r in rows):
                raise SkillError('MODEL_RESULT_UNKNOWN', '尚有未结算调用,禁止重放。', 409)
            pending=run.get('budget_request') or {}
            if pending.get('mode')=='overdraft':
                raise SkillError('OVERDRAFT_APPROVAL_PENDING','先行额度申请尚未决定,不叠加模型调用。',409)
            if overdraft:
                if run['request'].get('budget_policy')!={'mode':'one-call-overdraft-v1','increment':12000}:
                    raise SkillError('OVERDRAFT_NOT_AUTHORIZED','该任务没有先行执行授权。',403)
                check=overdraft['check'];stage=check['stage'];meta=metadata or {}
                stage_before=max(check['stage_limit'],run.get('approved_stage_limits',{}).get(stage,0))
                model_limit=overdraft.get('model_context_limit')
                if (amount<=0 or amount>run['budget']['max_tokens']+12000-used
                        or overdraft['stage_limit']>stage_before+12000 or amount>overdraft['stage_limit']
                        or (model_limit is not None and amount>model_limit)
                        or meta.get('requested_output_tokens',0)<overdraft['minimum_output']):
                    raise SkillError('OVERDRAFT_LIMIT_EXCEEDED','先行额度不足或触及模型硬窗口,未调用。',409)
                before=dict(run['budget'])
                after={**before,'max_tokens':before['max_tokens']+12000,
                    'max_model_calls':max(before['max_model_calls'],len(rows)+1),
                    'max_repairs':max(before['max_repairs'],run['repairs'])}
                proposal={'mode':'overdraft','stage':stage,'model':meta.get('model',provider),
                    'before':before,'after':after,'stage_before':stage_before,'stage_limit':overdraft['stage_limit'],
                    'token_increment':12000,'created_at':time.time(),'snapshot':run['snapshot']['snapshot_digest'],
                    'planned_call_reservation':amount,'output_reservation':meta['requested_output_tokens'],
                    'input_bound':meta['budget_check']['input_bound'],'call_number':len(rows)+1,
                    'task_revision':run['revision']+1}
                proposal['digest']=digest({'run':run_id,**proposal})
                run['budget_request']=proposal;run['pending_repair_extension']=False
                run['revision']+=1
                db.execute('UPDATE runs SET body=? WHERE id=?',(canonical(run),run_id))
                db.execute('INSERT INTO runtime_events(run_id,timestamp,action,body_digest) VALUES(?,?,?,?)',
                    (run_id,time.time(),'overdraft_reserved',digest(proposal)))
            unlimited=run['request'].get('budget_policy',{}).get('mode')=='unlimited-usage-v1'
            if not overdraft and not unlimited and len(rows)>=run['budget']['max_model_calls']:
                raise SkillError('MODEL_CALL_BUDGET_EXCEEDED','本任务实际模型请求次数已达上限。',409)
            if not overdraft and not unlimited and used+amount>run['budget']['max_tokens']:
                raise SkillError('TASK_TOKEN_BUDGET_EXCEEDED','本任务已结算及预留合计超过总 Token 预算。',409)
            cur = db.execute('INSERT INTO calls(run_id,reservation,state,provider,prompt_digest,metadata) VALUES(?,?,?,?,?,?)',
                             (run_id, amount, 'reserved', provider, digest(prompt), canonical({**(metadata or {}),'started_at':time.time()})))
            return cur.lastrowid

    def settle(self, call_id, state, charged=None, details=None):
        with self.connect() as db:
            row=db.execute('SELECT metadata FROM calls WHERE id=?',(call_id,)).fetchone()
            metadata=json.loads(row['metadata'])
            metadata.update(details or {})
            metadata['ended_at']=time.time()
            db.execute("UPDATE calls SET state=?,charged=?,metadata=? WHERE id=? AND state='reserved'", (state, charged, canonical(metadata), call_id))

    def calls(self, run_id):
        with self.connect() as db:
            return [{**dict(r),'metadata':json.loads(r['metadata'])} for r in db.execute('SELECT * FROM calls WHERE run_id=? ORDER BY id', (run_id,))]

    def stage_event(self, run_id, stage, status, detail=''):
        with self.connect() as db:
            db.execute('INSERT INTO runtime_stage_events(run_id,timestamp,stage,status,detail) VALUES(?,?,?,?,?)',
                       (run_id,time.time(),stage,status,str(detail)[:500]))

    def diagnostic(self, run_id, component, code, exception=None):
        # Only structural information is retained: never exception text, prompts or secrets.
        clean=lambda value: re.sub(r'[^A-Za-z0-9_.-]','_',str(value))[:100]
        frames=[]
        if exception is not None and exception.__traceback__ is not None:
            for frame in traceback.extract_tb(exception.__traceback__)[-8:]:
                frames.append({'file':clean(Path(frame.filename).name),'function':clean(frame.name),'line':frame.lineno})
        with self.connect() as db:
            db.execute('INSERT INTO runtime_diagnostics(run_id,timestamp,component,code,exception_type,frames) VALUES(?,?,?,?,?,?)',
                       (clean(run_id),time.time(),clean(component),clean(code),clean(type(exception).__name__) if exception else '',canonical(frames)))
            db.execute('DELETE FROM runtime_diagnostics WHERE id <= (SELECT coalesce(max(id),0)-10000 FROM runtime_diagnostics)')

    def diagnostics(self,run_id):
        with self.connect() as db:
            return [dict(row) for row in db.execute('SELECT * FROM runtime_diagnostics WHERE run_id=? ORDER BY id',(run_id,))]

    def stages(self, run_id):
        with self.connect() as db:
            return [dict(row) for row in db.execute('SELECT * FROM runtime_stage_events WHERE run_id=? ORDER BY id LIMIT 1000',(run_id,))]