Download source/runtime/launch.py from manateelazycat/Index-Translate-AIPod: direct link, hf CLI and curl.
- Browser
- Download file 4.09 kB
-
https://huggingface.co/manateelazycat/Index-Translate-AIPod/resolve/main/source/runtime/launch.py
- Command line
-
hf download hf://manateelazycat/Index-Translate-AIPod/source/runtime/launch.py
-
curl -L -o launch.py https://huggingface.co/manateelazycat/Index-Translate-AIPod/resolve/main/source/runtime/launch.py
4.09 kB
| """Start a fully installed offline runtime; weights stay on a read-only mount.""" | |
| import json | |
| import os | |
| import pathlib | |
| import sys | |
| backend = os.environ['INDEX_BACKEND'] | |
| model = pathlib.Path(os.environ['INDEX_MODEL_DIR']) | |
| model_id = os.environ['INDEX_MODEL_ID'] | |
| port = int(os.environ.get('INDEX_PORT', '18156')) | |
| context = int(os.environ.get('INDEX_CONTEXT_LENGTH', '32768')) | |
| concurrency = int(os.environ.get('INDEX_CONCURRENCY', '1')) | |
| profile = json.loads(os.environ.get('INDEX_PROFILE_JSON', '{}')) | |
| if not 1024 <= context <= 262144 or context % 1024 or not 1 <= concurrency <= 128: | |
| raise SystemExit('Invalid runtime limits') | |
| if not model.is_dir() or not (model / 'config.json').is_file(): | |
| raise SystemExit('The selected model snapshot is incomplete') | |
| os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', HF_DATASETS_OFFLINE='1') | |
| os.environ.update(VLLM_NO_USAGE_STATS='1', VLLM_DO_NOT_TRACK='1', HF_HUB_DISABLE_TELEMETRY='1') | |
| if backend == 'orinfer': | |
| os.environ['ORINFER_EXECUTION_CACHE'] = '/runtime/packages' | |
| os.environ['ORINFER_CONTEXT_LIMIT'] = str(context) | |
| command = ['/runtime/bin/orinfer', 'serve', str(model), '--model', model_id, | |
| '--listen', '0.0.0.0:' + str(port), '--max-active-requests', str(concurrency), | |
| '--gpu-lock', '/root/.cache/index-translate-gpu.lock', | |
| '--default-request-params', '{"enable_thinking":false}'] | |
| options = {'cuda_graph': ('--cuda-graph', 'decode_only'), 'mtp_drafts': ('--mtp-drafts', 0), | |
| 'prefix_cache_mib': ('--prefix-cache-mib', 0), 'load_workers': ('--load-workers', 4), | |
| 'scheduler_policy': ('--scheduler-policy', 'latency'), | |
| 'max_batch_tokens': ('--max-batch-tokens', 128), | |
| 'prefill_budget_ms': ('--prefill-budget-ms', 200), | |
| 'target_tpot_ms': ('--target-tpot-ms', 400), | |
| 'preprocess_workers': ('--preprocess-workers', 2)} | |
| if set(profile) - set(options): | |
| raise SystemExit('Unknown Orinfer profile field') | |
| for key, (flag, default) in options.items(): | |
| command += [flag, str(profile.get(key, default))] | |
| elif backend == 'vllm': | |
| options = {'mtp_drafts', 'prefix_cache', 'enforce_eager', 'max_num_batched_tokens', | |
| 'gpu_memory_utilization', 'kv_cache_dtype', 'attention_backend', 'async_scheduling'} | |
| if set(profile) - options: | |
| raise SystemExit('Unknown vLLM profile field') | |
| # The v0.29 ARM64 wheel omits an SM110 image for fused CUDA GDN MTP. | |
| # Keep the verified Triton implementation for speculative decode on Thor. | |
| if profile.get('mtp_drafts', 0): | |
| os.environ['VLLM_GDN_DECODE_KERNEL'] = 'triton' | |
| command = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', str(model), | |
| '--served-model-name', model_id, '--host', '0.0.0.0', '--port', str(port), | |
| '--max-model-len', str(context), '--max-num-seqs', str(concurrency), | |
| '--gpu-memory-utilization', str(profile.get('gpu_memory_utilization', .65)), | |
| '--max-num-batched-tokens', str(profile.get('max_num_batched_tokens', 4096)), | |
| '--default-chat-template-kwargs', '{"enable_thinking":false}', | |
| '--reasoning-parser', 'qwen3', '--no-enable-log-requests'] | |
| command += ['--enable-prefix-caching' if profile.get('prefix_cache', False) else '--no-enable-prefix-caching'] | |
| if profile.get('enforce_eager', True): | |
| command += ['--enforce-eager'] | |
| if profile.get('mtp_drafts', 0): | |
| command += ['--speculative-config', json.dumps({'method': 'mtp', 'num_speculative_tokens': profile['mtp_drafts']})] | |
| if profile.get('kv_cache_dtype'): | |
| command += ['--kv-cache-dtype', profile['kv_cache_dtype']] | |
| if profile.get('attention_backend'): | |
| command += ['--attention-backend', profile['attention_backend']] | |
| if 'async_scheduling' in profile: | |
| command += ['--async-scheduling' if profile['async_scheduling'] else '--no-async-scheduling'] | |
| else: | |
| raise SystemExit('Unsupported runtime backend') | |
| os.execv(command[0], command) | |