BaseDecision-Dyno / serve_typesafe.py
onlyaady's picture
Publish calibrated BaseDecision-Dyno with portable HTTP serving
3e88dcc verified
Raw History Blame Contribute Delete
12.1 kB
"""Resident, one-GPU TypeSafe-wire server. No prompt logging or automatic fallback."""
import argparse
import gzip
import hmac
import io
import json
import math
import os
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import threading
MAX_BODY = 32 * 1024 * 1024
class RequestError(ValueError):
def __init__(self, code, message, status=422):
super().__init__(message); self.code = code; self.status = status
def validate_json(value, depth=0):
if depth > 64: raise RequestError('invalid_json', 'JSON nesting exceeds 64', 400)
if isinstance(value, float) and not math.isfinite(value):
raise RequestError('invalid_json', 'Finite numbers required', 400)
if isinstance(value, (dict, list)):
for item in value.values() if isinstance(value, dict) else value:
validate_json(item, depth + 1)
def unique_object(pairs):
result = {}
for key, value in pairs:
if key in result: raise RequestError('invalid_json', 'Duplicate JSON key', 400)
result[key] = value
return result
def check_body(body, mode, model_id):
if not isinstance(body, dict) or 'state' not in body:
raise RequestError('invalid_request', 'state and questions are required')
if not isinstance(body['state'], (str, dict, list)):
raise RequestError('invalid_request', 'state must be a string, object or array')
if body.get('images') or body.get('assets'):
raise RequestError('unsupported_modality', 'Text-only model')
if body.get('model') not in (model_id, 'BaseDecision-Dyno', 'onlyaady/BaseDecision-Dyno'):
raise RequestError('model_not_found', 'Use the model id returned by GET /v1/models', 404)
if body.get('mode', mode) != mode:
raise RequestError('mode_mismatch', 'This process has a fixed mode; flash requires an explicitly launched flash server')
questions = body.get('questions')
if not isinstance(questions, dict) or not 1 <= len(questions) <= 64:
raise RequestError('invalid_questions', 'Supply 1..64 questions keyed by caller ids')
for qid, q in questions.items():
if not qid or not isinstance(q, dict): raise RequestError('invalid_question', 'Invalid question entry')
kind = q.get('type'); criteria = q.get('criteria')
if kind == 'choice':
if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 255:
raise RequestError('invalid_criteria', 'choice requires 2..255 options')
elif kind == 'score':
if not isinstance(criteria, list) or not 2 <= len(criteria) <= 10:
raise RequestError('invalid_criteria', 'score requires 2..10 ordered levels')
elif kind == 'noul':
if criteria is not None and (not isinstance(criteria, dict) or set(criteria) - {'true', 'false'}):
raise RequestError('invalid_criteria', 'noul criteria use true/false keys')
else: raise RequestError('unsupported_question_type', 'Supported: choice, noul, score')
return {'state': body['state'], 'questions': questions}
def wire_response(request, native, model_id):
answers = {}
for qid, question in request['questions'].items():
value = native['answers'][qid]; probabilities = value['probabilities']
kind = question['type']
expected = (list(question['criteria']) if kind == 'choice' else
['true', 'false'] if kind == 'noul' else
[str(i) for i in range(len(question['criteria']))])
if set(probabilities) != set(expected): raise RuntimeError('Native option mapping differs')
if any(isinstance(p, bool) or not isinstance(p, (int, float)) or not math.isfinite(p) or not 0 <= p <= 1 for p in probabilities.values()):
raise RuntimeError('Invalid native probability')
if abs(sum(probabilities.values()) - 1) > 1e-5: raise RuntimeError('Invalid probability sum')
if kind == 'noul': answers[qid] = {'type': kind, 'noul': probabilities['true']}
elif kind == 'choice':
answers[qid] = {'type': kind, 'choice': value['choice'], 'probabilities': probabilities}
else:
answers[qid] = {'type': kind, 'score': sum(int(k) * p for k, p in probabilities.items()),
'probabilities': probabilities,
'legend': {str(i): v for i, v in enumerate(question['criteria'])}}
return {'model': model_id, 'answers': answers,
'usage': {'input_tokens': sum(native['sequence_tokens']), 'output_tokens': 0},
'mode': native['mode'], 'read_twice': native['read_twice'],
'calibrated': native['calibrated'], 'release_id': native['release_id']}
class Application:
def __init__(self, model, mode='default', key=None, synchronize=lambda: None):
self.model = model; self.mode = mode; self.key = key
self.model_id = 'onlyaady/BaseDecision-Dyno' if mode == 'default' else 'onlyaady/BaseDecision-Dyno-flash'
self.synchronize = synchronize; self.lock = threading.Lock()
def decide(self, body):
request = check_body(body, self.mode, self.model_id)
if not self.lock.acquire(blocking=False):
raise RequestError('overloaded', 'One request is in progress', 529)
try:
result = self.model.decide(request, mode=self.mode)
self.synchronize()
return wire_response(request, result, self.model_id)
except ValueError as exc:
if type(exc).__name__ == 'LengthError':
raise RequestError('max_tokens_exceeded', 'Rendered question exceeds 16,384 tokens including repetition; no truncation or flash fallback', 400) from exc
raise RequestError('invalid_request', str(exc)[:300]) from exc
finally: self.lock.release()
def handler_for(app):
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args): pass # No question/state/auth data in server logs.
def send_json(self, status, payload):
raw = json.dumps(payload, ensure_ascii=False, allow_nan=False).encode()
self.send_response(status); self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(raw))); self.send_header('Connection', 'close')
self.end_headers(); self.wfile.write(raw); self.close_connection = True
def authorized(self):
return app.key is None or hmac.compare_digest(self.headers.get('Authorization', ''), 'Bearer ' + app.key)
def do_GET(self):
if self.path == '/health':
self.send_json(200, {'ready': True, 'model': app.model_id, 'mode': app.mode,
'release_id': app.model.config.release_id, 'rendered_context_limit': 16384,
'calibrated': app.model.config.calibrated}); return
if not self.authorized(): self.send_json(401, {'error': {'code': 'unauthorized'}}); return
if self.path == '/v1/models':
self.send_json(200, {'models': [{'id': app.model_id, 'context_length': 16384,
'question_types': ['choice', 'noul', 'score'], 'read_twice': app.mode == 'default'}]})
else: self.send_json(404, {'error': {'code': 'not_found'}})
def do_POST(self):
try:
if self.path != '/v1/systemone': raise RequestError('not_found', 'Use /v1/systemone', 404)
if not self.authorized(): raise RequestError('unauthorized', 'Invalid bearer token', 401)
if self.headers.get('Transfer-Encoding'): raise RequestError('invalid_request', 'Content-Length required', 400)
if self.headers.get('Content-Type', '').split(';')[0].strip().lower() != 'application/json':
raise RequestError('unsupported_media_type', 'Send application/json', 415)
try: size = int(self.headers.get('Content-Length', '-1'))
except ValueError: size = -1
if size < 0: raise RequestError('invalid_request', 'Content-Length required', 400)
if size > MAX_BODY: raise RequestError('body_too_large', 'Body exceeds 32 MiB', 413)
self.connection.settimeout(30); raw = self.rfile.read(size)
if len(raw) != size: raise RequestError('invalid_request', 'Incomplete body', 400)
coding = self.headers.get('Content-Encoding', 'identity').strip().lower()
if coding in ('gzip', 'x-gzip'):
try:
with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream: raw = stream.read(MAX_BODY + 1)
except (OSError, EOFError): raise RequestError('invalid_json', 'Malformed gzip body', 400)
if len(raw) > MAX_BODY: raise RequestError('body_too_large', 'Decompressed body exceeds 32 MiB', 413)
elif coding != 'identity': raise RequestError('unsupported_encoding', 'Supported encodings: identity, gzip', 415)
try: body = json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object)
except (UnicodeError, json.JSONDecodeError, RecursionError): raise RequestError('invalid_json', 'Invalid UTF-8 JSON', 400)
validate_json(body); self.send_json(200, app.decide(body))
except RequestError as exc:
self.send_json(exc.status, {'error': {'code': exc.code, 'message': str(exc)}})
except Exception:
self.send_json(500, {'error': {'code': 'inference_error', 'message': 'Inference failed; no fallback answer produced'}})
return Handler
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--model', default=str(Path(__file__).resolve().parent))
p.add_argument('--revision'); p.add_argument('--mode', choices=('default', 'flash'), default='default')
p.add_argument('--host', default='127.0.0.1'); p.add_argument('--port', type=int, default=8000)
p.add_argument('--gpu', type=int, default=0); p.add_argument('--threads', type=int, default=4)
p.add_argument('--key-env', default='DYNO_API_KEY')
a = p.parse_args()
if a.gpu < 0 or a.threads < 1: raise ValueError('Invalid GPU/thread count')
devices = os.environ.get('CUDA_VISIBLE_DEVICES')
if devices is not None:
values = [s.strip() for s in devices.split(',') if s.strip()]
if a.gpu >= len(values): raise ValueError('GPU index outside allocation')
os.environ['CUDA_VISIBLE_DEVICES'] = values[a.gpu]
else: os.environ['CUDA_VISIBLE_DEVICES'] = str(a.gpu)
import torch
from transformers import AutoModel
torch.set_num_threads(a.threads); torch.cuda.set_device(0)
torch.backends.cuda.enable_cudnn_sdp(False)
model = AutoModel.from_pretrained(a.model, revision=a.revision, trust_remote_code=True,
local_files_only=Path(a.model).is_dir(), dtype=torch.bfloat16, device_map={'': 'cuda:0'}).eval()
if model.config.max_length != 16384 or model.config.protocol != 'dyno-strict-read-twice-v1':
raise ValueError('Wrong model protocol/context')
_ = model.processor
# Warm all native decision readouts once; warmup is outside request measurements.
for question in ({'type': 'choice', 'criteria': {'a': 'first', 'b': 'second'}},
{'type': 'noul'}, {'type': 'score', 'criteria': ['low', 'high']}):
model.decide({'state': 'Server warmup.', 'question': question}, mode=a.mode)
torch.cuda.synchronize()
app = Application(model, a.mode, os.environ.get(a.key_env) if a.key_env else None, torch.cuda.synchronize)
server = ThreadingHTTPServer((a.host, a.port), handler_for(app)); server.daemon_threads = True
print(json.dumps({'ready': True, 'model': app.model_id, 'mode': a.mode, 'host': a.host,
'port': a.port, 'release_id': model.config.release_id}), flush=True)
try: server.serve_forever()
except KeyboardInterrupt: pass
finally: server.server_close()
if __name__ == '__main__': main()