Spaces:
Sleeping
Sleeping
File size: 10,767 Bytes
caf8059 5ca3c16 caf8059 5ca3c16 caf8059 115b94d caf8059 115b94d caf8059 115b94d caf8059 115b94d caf8059 5ca3c16 caf8059 5ca3c16 caf8059 5ca3c16 caf8059 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Atomic local run state, fenced claims and durable model reservations."""
import json
import sqlite3
import time
import re
import traceback
from contextlib import contextmanager
from pathlib import Path
from src.modules.skill.domain.models import SkillError, canonical, digest
class RuntimeStore:
def __init__(self, path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.connect() as db:
db.executescript('''
CREATE TABLE IF NOT EXISTS runs(id TEXT PRIMARY KEY, tenant TEXT NOT NULL, owner TEXT NOT NULL,
request_key TEXT NOT NULL, request_digest TEXT NOT NULL, body TEXT NOT NULL,
UNIQUE(tenant,owner,request_key));
CREATE TABLE IF NOT EXISTS calls(id INTEGER PRIMARY KEY, run_id TEXT NOT NULL, reservation INTEGER NOT NULL,
charged INTEGER, state TEXT NOT NULL, provider TEXT NOT NULL, prompt_digest TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS runtime_events(id INTEGER PRIMARY KEY, run_id TEXT NOT NULL,
timestamp REAL NOT NULL, action TEXT NOT NULL, body_digest TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS runtime_stage_events(id INTEGER PRIMARY KEY,run_id TEXT NOT NULL,
timestamp REAL NOT NULL,stage TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS runtime_diagnostics(id INTEGER PRIMARY KEY,run_id TEXT NOT NULL,
timestamp REAL NOT NULL,component TEXT NOT NULL,code TEXT NOT NULL,exception_type TEXT NOT NULL,frames TEXT NOT NULL);
''')
if 'metadata' not in {r['name'] for r in db.execute('PRAGMA table_info(calls)')}:
db.execute("ALTER TABLE calls ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'")
@contextmanager
def connect(self):
db = sqlite3.connect(self.path, timeout=30)
db.row_factory = sqlite3.Row
try:
db.execute('BEGIN IMMEDIATE')
yield db
db.commit()
except BaseException:
db.rollback()
raise
finally:
db.close()
def create(self, run, key, request):
with self.connect() as db:
old = db.execute('SELECT * FROM runs WHERE tenant=? AND owner=? AND request_key=?',
(run['tenant_id'], run['owner_id'], key)).fetchone()
if old:
if old['request_digest'] != digest(request):
raise SkillError('IDEMPOTENCY_CONFLICT', '同一请求键输入发生变化。', 409)
return json.loads(old['body'])
db.execute('INSERT INTO runs VALUES(?,?,?,?,?,?)', (run['id'], run['tenant_id'], run['owner_id'], key, digest(request), canonical(run)))
return run
def get(self, run_id):
with self.connect() as db:
row = db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()
if not row:
raise SkillError('RUN_NOT_FOUND', '任务不存在或不可见。', 404)
return json.loads(row['body'])
def mutate(self, run_id, fn, *, with_calls=False):
with self.connect() as db:
row = db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()
if not row:
raise SkillError('RUN_NOT_FOUND', '任务不存在或不可见。', 404)
body = json.loads(row['body'])
if with_calls:
# Approval validates the ledger under the same write transaction.
# Idempotent decisions must not change the task version.
before=canonical(body)
calls=[{**dict(r),'metadata':json.loads(r['metadata'])} for r in db.execute('SELECT * FROM calls WHERE run_id=? ORDER BY id',(run_id,))]
fn(body,calls)
if canonical(body)==before:return body
else:fn(body)
body['revision'] += 1
db.execute('UPDATE runs SET body=? WHERE id=?', (canonical(body), run_id))
db.execute('INSERT INTO runtime_events(run_id,timestamp,action,body_digest) VALUES(?,?,?,?)',
(run_id, time.time(), body['status'], digest(body)))
return body
def reserve(self, run_id, provider, prompt, amount, metadata=None, *, overdraft=None):
with self.connect() as db:
run = json.loads(db.execute('SELECT body FROM runs WHERE id=?', (run_id,)).fetchone()['body'])
if run['status'] != 'running' or run.get('cancel_requested'):
raise SkillError('RUN_NOT_ACTIVE', '任务不允许调用模型。', 409)
rows = db.execute('SELECT * FROM calls WHERE run_id=?', (run_id,)).fetchall()
used = sum(r['charged'] if r['charged'] is not None else r['reservation'] for r in rows)
if any(r['state'] in {'reserved', 'unknown'} for r in rows):
raise SkillError('MODEL_RESULT_UNKNOWN', '尚有未结算调用,禁止重放。', 409)
pending=run.get('budget_request') or {}
if pending.get('mode')=='overdraft':
raise SkillError('OVERDRAFT_APPROVAL_PENDING','先行额度申请尚未决定,不叠加模型调用。',409)
if overdraft:
if run['request'].get('budget_policy')!={'mode':'one-call-overdraft-v1','increment':12000}:
raise SkillError('OVERDRAFT_NOT_AUTHORIZED','该任务没有先行执行授权。',403)
check=overdraft['check'];stage=check['stage'];meta=metadata or {}
stage_before=max(check['stage_limit'],run.get('approved_stage_limits',{}).get(stage,0))
model_limit=overdraft.get('model_context_limit')
if (amount<=0 or amount>run['budget']['max_tokens']+12000-used
or overdraft['stage_limit']>stage_before+12000 or amount>overdraft['stage_limit']
or (model_limit is not None and amount>model_limit)
or meta.get('requested_output_tokens',0)<overdraft['minimum_output']):
raise SkillError('OVERDRAFT_LIMIT_EXCEEDED','先行额度不足或触及模型硬窗口,未调用。',409)
before=dict(run['budget'])
after={**before,'max_tokens':before['max_tokens']+12000,
'max_model_calls':max(before['max_model_calls'],len(rows)+1),
'max_repairs':max(before['max_repairs'],run['repairs'])}
proposal={'mode':'overdraft','stage':stage,'model':meta.get('model',provider),
'before':before,'after':after,'stage_before':stage_before,'stage_limit':overdraft['stage_limit'],
'token_increment':12000,'created_at':time.time(),'snapshot':run['snapshot']['snapshot_digest'],
'planned_call_reservation':amount,'output_reservation':meta['requested_output_tokens'],
'input_bound':meta['budget_check']['input_bound'],'call_number':len(rows)+1,
'task_revision':run['revision']+1}
proposal['digest']=digest({'run':run_id,**proposal})
run['budget_request']=proposal;run['pending_repair_extension']=False
run['revision']+=1
db.execute('UPDATE runs SET body=? WHERE id=?',(canonical(run),run_id))
db.execute('INSERT INTO runtime_events(run_id,timestamp,action,body_digest) VALUES(?,?,?,?)',
(run_id,time.time(),'overdraft_reserved',digest(proposal)))
unlimited=run['request'].get('budget_policy',{}).get('mode')=='unlimited-usage-v1'
if not overdraft and not unlimited and len(rows)>=run['budget']['max_model_calls']:
raise SkillError('MODEL_CALL_BUDGET_EXCEEDED','本任务实际模型请求次数已达上限。',409)
if not overdraft and not unlimited and used+amount>run['budget']['max_tokens']:
raise SkillError('TASK_TOKEN_BUDGET_EXCEEDED','本任务已结算及预留合计超过总 Token 预算。',409)
cur = db.execute('INSERT INTO calls(run_id,reservation,state,provider,prompt_digest,metadata) VALUES(?,?,?,?,?,?)',
(run_id, amount, 'reserved', provider, digest(prompt), canonical({**(metadata or {}),'started_at':time.time()})))
return cur.lastrowid
def settle(self, call_id, state, charged=None, details=None):
with self.connect() as db:
row=db.execute('SELECT metadata FROM calls WHERE id=?',(call_id,)).fetchone()
metadata=json.loads(row['metadata'])
metadata.update(details or {})
metadata['ended_at']=time.time()
db.execute("UPDATE calls SET state=?,charged=?,metadata=? WHERE id=? AND state='reserved'", (state, charged, canonical(metadata), call_id))
def calls(self, run_id):
with self.connect() as db:
return [{**dict(r),'metadata':json.loads(r['metadata'])} for r in db.execute('SELECT * FROM calls WHERE run_id=? ORDER BY id', (run_id,))]
def stage_event(self, run_id, stage, status, detail=''):
with self.connect() as db:
db.execute('INSERT INTO runtime_stage_events(run_id,timestamp,stage,status,detail) VALUES(?,?,?,?,?)',
(run_id,time.time(),stage,status,str(detail)[:500]))
def diagnostic(self, run_id, component, code, exception=None):
# Only structural information is retained: never exception text, prompts or secrets.
clean=lambda value: re.sub(r'[^A-Za-z0-9_.-]','_',str(value))[:100]
frames=[]
if exception is not None and exception.__traceback__ is not None:
for frame in traceback.extract_tb(exception.__traceback__)[-8:]:
frames.append({'file':clean(Path(frame.filename).name),'function':clean(frame.name),'line':frame.lineno})
with self.connect() as db:
db.execute('INSERT INTO runtime_diagnostics(run_id,timestamp,component,code,exception_type,frames) VALUES(?,?,?,?,?,?)',
(clean(run_id),time.time(),clean(component),clean(code),clean(type(exception).__name__) if exception else '',canonical(frames)))
db.execute('DELETE FROM runtime_diagnostics WHERE id <= (SELECT coalesce(max(id),0)-10000 FROM runtime_diagnostics)')
def diagnostics(self,run_id):
with self.connect() as db:
return [dict(row) for row in db.execute('SELECT * FROM runtime_diagnostics WHERE run_id=? ORDER BY id',(run_id,))]
def stages(self, run_id):
with self.connect() as db:
return [dict(row) for row in db.execute('SELECT * FROM runtime_stage_events WHERE run_id=? ORDER BY id LIMIT 1000',(run_id,))]
|