manateelazycat's picture
Index Translate LPK 0.1.0: artifacts
0698504 verified
Raw History Blame Contribute Delete
4.09 kB
"""Start a fully installed offline runtime; weights stay on a read-only mount."""
import json
import os
import pathlib
import sys
backend = os.environ['INDEX_BACKEND']
model = pathlib.Path(os.environ['INDEX_MODEL_DIR'])
model_id = os.environ['INDEX_MODEL_ID']
port = int(os.environ.get('INDEX_PORT', '18156'))
context = int(os.environ.get('INDEX_CONTEXT_LENGTH', '32768'))
concurrency = int(os.environ.get('INDEX_CONCURRENCY', '1'))
profile = json.loads(os.environ.get('INDEX_PROFILE_JSON', '{}'))
if not 1024 <= context <= 262144 or context % 1024 or not 1 <= concurrency <= 128:
raise SystemExit('Invalid runtime limits')
if not model.is_dir() or not (model / 'config.json').is_file():
raise SystemExit('The selected model snapshot is incomplete')
os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', HF_DATASETS_OFFLINE='1')
os.environ.update(VLLM_NO_USAGE_STATS='1', VLLM_DO_NOT_TRACK='1', HF_HUB_DISABLE_TELEMETRY='1')
if backend == 'orinfer':
os.environ['ORINFER_EXECUTION_CACHE'] = '/runtime/packages'
os.environ['ORINFER_CONTEXT_LIMIT'] = str(context)
command = ['/runtime/bin/orinfer', 'serve', str(model), '--model', model_id,
'--listen', '0.0.0.0:' + str(port), '--max-active-requests', str(concurrency),
'--gpu-lock', '/root/.cache/index-translate-gpu.lock',
'--default-request-params', '{"enable_thinking":false}']
options = {'cuda_graph': ('--cuda-graph', 'decode_only'), 'mtp_drafts': ('--mtp-drafts', 0),
'prefix_cache_mib': ('--prefix-cache-mib', 0), 'load_workers': ('--load-workers', 4),
'scheduler_policy': ('--scheduler-policy', 'latency'),
'max_batch_tokens': ('--max-batch-tokens', 128),
'prefill_budget_ms': ('--prefill-budget-ms', 200),
'target_tpot_ms': ('--target-tpot-ms', 400),
'preprocess_workers': ('--preprocess-workers', 2)}
if set(profile) - set(options):
raise SystemExit('Unknown Orinfer profile field')
for key, (flag, default) in options.items():
command += [flag, str(profile.get(key, default))]
elif backend == 'vllm':
options = {'mtp_drafts', 'prefix_cache', 'enforce_eager', 'max_num_batched_tokens',
'gpu_memory_utilization', 'kv_cache_dtype', 'attention_backend', 'async_scheduling'}
if set(profile) - options:
raise SystemExit('Unknown vLLM profile field')
# The v0.29 ARM64 wheel omits an SM110 image for fused CUDA GDN MTP.
# Keep the verified Triton implementation for speculative decode on Thor.
if profile.get('mtp_drafts', 0):
os.environ['VLLM_GDN_DECODE_KERNEL'] = 'triton'
command = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', str(model),
'--served-model-name', model_id, '--host', '0.0.0.0', '--port', str(port),
'--max-model-len', str(context), '--max-num-seqs', str(concurrency),
'--gpu-memory-utilization', str(profile.get('gpu_memory_utilization', .65)),
'--max-num-batched-tokens', str(profile.get('max_num_batched_tokens', 4096)),
'--default-chat-template-kwargs', '{"enable_thinking":false}',
'--reasoning-parser', 'qwen3', '--no-enable-log-requests']
command += ['--enable-prefix-caching' if profile.get('prefix_cache', False) else '--no-enable-prefix-caching']
if profile.get('enforce_eager', True):
command += ['--enforce-eager']
if profile.get('mtp_drafts', 0):
command += ['--speculative-config', json.dumps({'method': 'mtp', 'num_speculative_tokens': profile['mtp_drafts']})]
if profile.get('kv_cache_dtype'):
command += ['--kv-cache-dtype', profile['kv_cache_dtype']]
if profile.get('attention_backend'):
command += ['--attention-backend', profile['attention_backend']]
if 'async_scheduling' in profile:
command += ['--async-scheduling' if profile['async_scheduling'] else '--no-async-scheduling']
else:
raise SystemExit('Unsupported runtime backend')
os.execv(command[0], command)