Dexter commited on
Commit
569d24c
·
verified ·
1 Parent(s): 8eb31c2

Upload mirothinker server.py

Browse files
Files changed (1) hide show
  1. mirothinker server.py +119 -0
mirothinker server.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Minimal OpenAI-compatible server for MiroThinker using plain transformers.
2
+ # No vLLM/SGLang needed. Works wherever PyTorch CUDA works (incl. Windows).
3
+ #
4
+ # pip install transformers accelerate fastapi uvicorn torch
5
+ # python mirothinker_server.py
6
+ #
7
+ # Then set in miroflow-agent .env / config:
8
+ # base_url: http://localhost:61005/v1 (provider "qwen", any api_key)
9
+ #
10
+ # Handles the two non-standard params miroflow-agent sends via extra_body:
11
+ # - repetition_penalty
12
+ # - continue_final_message / add_generation_prompt (resume truncated output)
13
+
14
+ import time
15
+ import uuid
16
+
17
+ import torch
18
+ import uvicorn
19
+ from fastapi import FastAPI
20
+ from pydantic import BaseModel
21
+ from transformers import AutoModelForCausalLM, AutoTokenizer
22
+
23
+ MODEL_ID = "miromind-ai/MiroThinker-v1.5-30B" # change if using 1.7 etc.
24
+ PORT = 61005
25
+
26
+ print(f"Loading {MODEL_ID} ... (this takes a while)")
27
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
28
+ model = AutoModelForCausalLM.from_pretrained(
29
+ MODEL_ID,
30
+ torch_dtype=torch.bfloat16,
31
+ device_map="auto", # splits layers across both A100s
32
+ trust_remote_code=True,
33
+ )
34
+ model.eval()
35
+ print("Model loaded. Device map:", model.hf_device_map if hasattr(model, "hf_device_map") else "n/a")
36
+
37
+ app = FastAPI()
38
+
39
+
40
+ class ChatRequest(BaseModel):
41
+ model: str = MODEL_ID
42
+ messages: list
43
+ max_tokens: int | None = 16384
44
+ max_completion_tokens: int | None = None
45
+ temperature: float = 1.0
46
+ top_p: float = 0.95
47
+ stream: bool = False
48
+ # extra_body params arrive as top-level fields with the openai SDK
49
+ repetition_penalty: float | None = None
50
+ continue_final_message: bool | None = None
51
+ add_generation_prompt: bool | None = None
52
+
53
+ class Config:
54
+ extra = "allow"
55
+
56
+
57
+ @app.get("/health")
58
+ def health():
59
+ return {"status": "ok"}
60
+
61
+
62
+ @app.get("/v1/models")
63
+ def models():
64
+ return {"object": "list", "data": [{"id": MODEL_ID, "object": "model"}]}
65
+
66
+
67
+ @app.post("/v1/chat/completions")
68
+ def chat(req: ChatRequest):
69
+ continue_final = bool(req.continue_final_message)
70
+
71
+ # Build the prompt with the model's own chat template.
72
+ input_ids = tokenizer.apply_chat_template(
73
+ req.messages,
74
+ add_generation_prompt=not continue_final,
75
+ continue_final_message=continue_final,
76
+ return_tensors="pt",
77
+ ).to(model.device)
78
+
79
+ prompt_tokens = input_ids.shape[-1]
80
+ max_new = req.max_completion_tokens or req.max_tokens or 16384
81
+
82
+ gen_kwargs = dict(
83
+ max_new_tokens=max_new,
84
+ do_sample=req.temperature > 0,
85
+ temperature=max(req.temperature, 1e-5),
86
+ top_p=req.top_p,
87
+ pad_token_id=tokenizer.eos_token_id,
88
+ )
89
+ if req.repetition_penalty and req.repetition_penalty != 1.0:
90
+ gen_kwargs["repetition_penalty"] = req.repetition_penalty
91
+
92
+ with torch.inference_mode():
93
+ out = model.generate(input_ids, **gen_kwargs)
94
+
95
+ new_tokens = out[0][prompt_tokens:]
96
+ text = tokenizer.decode(new_tokens, skip_special_tokens=True)
97
+ completion_tokens = new_tokens.shape[-1]
98
+ finish = "length" if completion_tokens >= max_new else "stop"
99
+
100
+ return {
101
+ "id": f"chatcmpl-{uuid.uuid4().hex[:12]}",
102
+ "object": "chat.completion",
103
+ "created": int(time.time()),
104
+ "model": req.model,
105
+ "choices": [{
106
+ "index": 0,
107
+ "message": {"role": "assistant", "content": text},
108
+ "finish_reason": finish,
109
+ }],
110
+ "usage": {
111
+ "prompt_tokens": prompt_tokens,
112
+ "completion_tokens": completion_tokens,
113
+ "total_tokens": prompt_tokens + completion_tokens,
114
+ },
115
+ }
116
+
117
+
118
+ if __name__ == "__main__":
119
+ uvicorn.run(app, host="0.0.0.0", port=PORT)