Download serving/serve_example.py from kitaniai/clover-2: direct link, hf CLI and curl.
- Browser
- Download file 3.7 kB
-
https://huggingface.co/kitaniai/clover-2/resolve/main/serving/serve_example.py
- Command line
-
hf download hf://kitaniai/clover-2/serving/serve_example.py
-
curl -L -o serve_example.py https://huggingface.co/kitaniai/clover-2/resolve/main/serving/serve_example.py
3.7 kB
| """Minimal example: load Clover 2's base model, attach the 7 trained LoRA experts and the trained router for | |
| real per-token dynamic routing, then generate. | |
| Requires: vllm, torch. Tested against vllm==0.31.0. | |
| IMPORTANT -- compilation_config: torch.compile silently drops every LoRA delta under CUDA graph capture | |
| unless it's disabled this way. Stubbing GPUModelRunner.capture_model only defers the CUDA-graph-REPLAY-buffer | |
| step; it does NOT defer torch.compile's own Dynamo tracing, which vLLM triggers during LLM()'s own internal | |
| profiling/warmup -- i.e. BEFORE attach() ever runs, against the ORIGINAL unpatched mlp. That compiled graph | |
| then gets reused by compile_or_warm_up_model() regardless of the live monkey-patch, silently dropping every | |
| LoRA delta. Disabling torch.compile (mode=0) and relying on vLLM's CUDA graph capture directly sidesteps this | |
| entirely: the capture-time warmup call then executes the patched Python forward directly. | |
| """ | |
| import os | |
| os.environ['VLLM_ENABLE_V1_MULTIPROCESSING'] = '0' | |
| import torch | |
| import vllm.v1.worker.gpu.model_runner as MR | |
| # capture_model must be stubbed out BEFORE LLM() builds the model, so graph capture doesn't run against the | |
| # original (unpatched) mlp. Restored and called manually after attach()+load_adapter(), below. | |
| _orig_capture_model = MR.GPUModelRunner.capture_model | |
| MR.GPUModelRunner.capture_model = lambda self, *a, **kw: 0 | |
| from vllm import LLM, SamplingParams | |
| import vllm_moelora as V | |
| BASE_MODEL = "./base" # path to the base checkpoint (this repo's base/ subfolder) | |
| ADAPTERS = { | |
| 1: "./adapters/roleplay/adapter_model.safetensors", | |
| 2: "./adapters/clover/adapter_model.safetensors", | |
| 3: "./adapters/security/adapter_model.safetensors", | |
| 4: "./adapters/math/adapter_model.safetensors", | |
| 5: "./adapters/longdoc2/adapter_model.safetensors", | |
| 6: "./adapters/efficiency/adapter_model.safetensors", | |
| 7: "./adapters/knowledge/adapter_model.safetensors", | |
| } | |
| ROUTER_PATH = "./router/trained_router_v2.pt" | |
| # Security (slot 3) is masked off by default -- enabling it measurably increases compliance with offensive | |
| # (dual-use) security-research requests. Remove 3 from this list to re-enable it. | |
| DEFAULT_MASKED_SLOTS = [3] | |
| def load_safetensors_adapter(path): | |
| from safetensors.torch import load_file | |
| return load_file(path) | |
| def main(): | |
| llm = LLM( | |
| model=BASE_MODEL, | |
| dtype="bfloat16", | |
| gpu_memory_utilization=0.85, | |
| trust_remote_code=False, | |
| additional_config={"gdn_prefill_backend": "triton"}, | |
| compilation_config={"mode": 0, "cudagraph_mode": "FULL"}, | |
| ) | |
| worker = llm.llm_engine.model_executor.driver_worker.worker | |
| model = worker.model_runner.model | |
| moles = V.attach(model, n_experts=7, rank=128, scaling=2.0, topk=2) | |
| for slot, path in ADAPTERS.items(): | |
| V.load_adapter(moles, slot, load_safetensors_adapter(path)) | |
| router_state = torch.load(ROUTER_PATH, map_location="cpu", weights_only=False) | |
| for layer, m in moles.items(): | |
| sd = router_state[layer] | |
| m.router.weight.data.copy_(sd["weight"].to(m.router.weight.device, m.router.weight.dtype)) | |
| m.router.bias.data.copy_(sd["bias"].to(m.router.bias.device, m.router.bias.dtype)) | |
| V.set_mask(moles, DEFAULT_MASKED_SLOTS) | |
| MR.GPUModelRunner.capture_model = _orig_capture_model | |
| worker.compile_or_warm_up_model() | |
| sp = SamplingParams(temperature=0.8, max_tokens=300) | |
| outs = llm.chat( | |
| [[{"role": "user", "content": "Write a short scene where a detective questions a nervous witness."}]], | |
| sp, | |
| chat_template_kwargs={"enable_thinking": False}, | |
| ) | |
| print(outs[0].outputs[0].text) | |
| if __name__ == "__main__": | |
| main() | |