"""Minimal example: load Clover 2's base model, attach the 7 trained LoRA experts and the trained router for real per-token dynamic routing, then generate. Requires: vllm, torch. Tested against vllm==0.31.0. IMPORTANT -- compilation_config: torch.compile silently drops every LoRA delta under CUDA graph capture unless it's disabled this way. Stubbing GPUModelRunner.capture_model only defers the CUDA-graph-REPLAY-buffer step; it does NOT defer torch.compile's own Dynamo tracing, which vLLM triggers during LLM()'s own internal profiling/warmup -- i.e. BEFORE attach() ever runs, against the ORIGINAL unpatched mlp. That compiled graph then gets reused by compile_or_warm_up_model() regardless of the live monkey-patch, silently dropping every LoRA delta. Disabling torch.compile (mode=0) and relying on vLLM's CUDA graph capture directly sidesteps this entirely: the capture-time warmup call then executes the patched Python forward directly. """ import os os.environ['VLLM_ENABLE_V1_MULTIPROCESSING'] = '0' import torch import vllm.v1.worker.gpu.model_runner as MR # capture_model must be stubbed out BEFORE LLM() builds the model, so graph capture doesn't run against the # original (unpatched) mlp. Restored and called manually after attach()+load_adapter(), below. _orig_capture_model = MR.GPUModelRunner.capture_model MR.GPUModelRunner.capture_model = lambda self, *a, **kw: 0 from vllm import LLM, SamplingParams import vllm_moelora as V BASE_MODEL = "./base" # path to the base checkpoint (this repo's base/ subfolder) ADAPTERS = { 1: "./adapters/roleplay/adapter_model.safetensors", 2: "./adapters/clover/adapter_model.safetensors", 3: "./adapters/security/adapter_model.safetensors", 4: "./adapters/math/adapter_model.safetensors", 5: "./adapters/longdoc2/adapter_model.safetensors", 6: "./adapters/efficiency/adapter_model.safetensors", 7: "./adapters/knowledge/adapter_model.safetensors", } ROUTER_PATH = "./router/trained_router_v2.pt" # Security (slot 3) is masked off by default -- enabling it measurably increases compliance with offensive # (dual-use) security-research requests. Remove 3 from this list to re-enable it. DEFAULT_MASKED_SLOTS = [3] def load_safetensors_adapter(path): from safetensors.torch import load_file return load_file(path) def main(): llm = LLM( model=BASE_MODEL, dtype="bfloat16", gpu_memory_utilization=0.85, trust_remote_code=False, additional_config={"gdn_prefill_backend": "triton"}, compilation_config={"mode": 0, "cudagraph_mode": "FULL"}, ) worker = llm.llm_engine.model_executor.driver_worker.worker model = worker.model_runner.model moles = V.attach(model, n_experts=7, rank=128, scaling=2.0, topk=2) for slot, path in ADAPTERS.items(): V.load_adapter(moles, slot, load_safetensors_adapter(path)) router_state = torch.load(ROUTER_PATH, map_location="cpu", weights_only=False) for layer, m in moles.items(): sd = router_state[layer] m.router.weight.data.copy_(sd["weight"].to(m.router.weight.device, m.router.weight.dtype)) m.router.bias.data.copy_(sd["bias"].to(m.router.bias.device, m.router.bias.dtype)) V.set_mask(moles, DEFAULT_MASKED_SLOTS) MR.GPUModelRunner.capture_model = _orig_capture_model worker.compile_or_warm_up_model() sp = SamplingParams(temperature=0.8, max_tokens=300) outs = llm.chat( [[{"role": "user", "content": "Write a short scene where a detective questions a nervous witness."}]], sp, chat_template_kwargs={"enable_thinking": False}, ) print(outs[0].outputs[0].text) if __name__ == "__main__": main()