"""Start a fully installed offline runtime; weights stay on a read-only mount.""" import json import os import pathlib import sys backend = os.environ['INDEX_BACKEND'] model = pathlib.Path(os.environ['INDEX_MODEL_DIR']) model_id = os.environ['INDEX_MODEL_ID'] port = int(os.environ.get('INDEX_PORT', '18156')) context = int(os.environ.get('INDEX_CONTEXT_LENGTH', '32768')) concurrency = int(os.environ.get('INDEX_CONCURRENCY', '1')) profile = json.loads(os.environ.get('INDEX_PROFILE_JSON', '{}')) if not 1024 <= context <= 262144 or context % 1024 or not 1 <= concurrency <= 128: raise SystemExit('Invalid runtime limits') if not model.is_dir() or not (model / 'config.json').is_file(): raise SystemExit('The selected model snapshot is incomplete') os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', HF_DATASETS_OFFLINE='1') os.environ.update(VLLM_NO_USAGE_STATS='1', VLLM_DO_NOT_TRACK='1', HF_HUB_DISABLE_TELEMETRY='1') if backend == 'orinfer': os.environ['ORINFER_EXECUTION_CACHE'] = '/runtime/packages' os.environ['ORINFER_CONTEXT_LIMIT'] = str(context) command = ['/runtime/bin/orinfer', 'serve', str(model), '--model', model_id, '--listen', '0.0.0.0:' + str(port), '--max-active-requests', str(concurrency), '--gpu-lock', '/root/.cache/index-translate-gpu.lock', '--default-request-params', '{"enable_thinking":false}'] options = {'cuda_graph': ('--cuda-graph', 'decode_only'), 'mtp_drafts': ('--mtp-drafts', 0), 'prefix_cache_mib': ('--prefix-cache-mib', 0), 'load_workers': ('--load-workers', 4), 'scheduler_policy': ('--scheduler-policy', 'latency'), 'max_batch_tokens': ('--max-batch-tokens', 128), 'prefill_budget_ms': ('--prefill-budget-ms', 200), 'target_tpot_ms': ('--target-tpot-ms', 400), 'preprocess_workers': ('--preprocess-workers', 2)} if set(profile) - set(options): raise SystemExit('Unknown Orinfer profile field') for key, (flag, default) in options.items(): command += [flag, str(profile.get(key, default))] elif backend == 'vllm': options = {'mtp_drafts', 'prefix_cache', 'enforce_eager', 'max_num_batched_tokens', 'gpu_memory_utilization', 'kv_cache_dtype', 'attention_backend', 'async_scheduling'} if set(profile) - options: raise SystemExit('Unknown vLLM profile field') # The v0.29 ARM64 wheel omits an SM110 image for fused CUDA GDN MTP. # Keep the verified Triton implementation for speculative decode on Thor. if profile.get('mtp_drafts', 0): os.environ['VLLM_GDN_DECODE_KERNEL'] = 'triton' command = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', str(model), '--served-model-name', model_id, '--host', '0.0.0.0', '--port', str(port), '--max-model-len', str(context), '--max-num-seqs', str(concurrency), '--gpu-memory-utilization', str(profile.get('gpu_memory_utilization', .65)), '--max-num-batched-tokens', str(profile.get('max_num_batched_tokens', 4096)), '--default-chat-template-kwargs', '{"enable_thinking":false}', '--reasoning-parser', 'qwen3', '--no-enable-log-requests'] command += ['--enable-prefix-caching' if profile.get('prefix_cache', False) else '--no-enable-prefix-caching'] if profile.get('enforce_eager', True): command += ['--enforce-eager'] if profile.get('mtp_drafts', 0): command += ['--speculative-config', json.dumps({'method': 'mtp', 'num_speculative_tokens': profile['mtp_drafts']})] if profile.get('kv_cache_dtype'): command += ['--kv-cache-dtype', profile['kv_cache_dtype']] if profile.get('attention_backend'): command += ['--attention-backend', profile['attention_backend']] if 'async_scheduling' in profile: command += ['--async-scheduling' if profile['async_scheduling'] else '--no-async-scheduling'] else: raise SystemExit('Unsupported runtime backend') os.execv(command[0], command)