File size: 1,038 Bytes
00db36f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | from tensorrt_llm.commands.serve import get_llm_args, launch_server
if __name__ == '__main__':
llm_args, _ = get_llm_args(
model=r'D:\AI\models\Josiefied-Qwen3-4B-ablit-gptq-engine',
tokenizer=r'D:\AI\models\Josiefied-Qwen3-4B-abliterated-v1',
backend='tensorrt',
max_beam_width=1,
max_batch_size=1,
max_num_tokens=16384,
max_seq_len=16384,
tensor_parallel_size=1,
pipeline_parallel_size=1,
context_parallel_size=1,
moe_expert_parallel_size=None,
gpus_per_node=1,
free_gpu_memory_fraction=0.9,
kv_cache_dtype='auto',
num_postprocess_workers=0,
trust_remote_code=False,
revision=None,
reasoning_parser='qwen3',
fail_fast_on_attention_window_too_large=False,
otlp_traces_endpoint=None,
enable_chunked_prefill=True,
)
launch_server(
host='localhost',
port=5001,
llm_args=llm_args,
served_model_name='Josiefied-Qwen3-4B',
)
|