Text Generation
Transformers
Safetensors
PEFT
English
unsloth
qwen2
trl
lora
missionctrl
ai-oversight
conversational
Instructions to use Jit-fnc/missionctrl_env with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Jit-fnc/missionctrl_env with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Jit-fnc/missionctrl_env") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Jit-fnc/missionctrl_env", device_map="auto") - PEFT
How to use Jit-fnc/missionctrl_env with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Jit-fnc/missionctrl_env with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Jit-fnc/missionctrl_env" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/Jit-fnc/missionctrl_env
- SGLang
How to use Jit-fnc/missionctrl_env with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Jit-fnc/missionctrl_env" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Jit-fnc/missionctrl_env" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Unsloth Desktop
- Docker Model Runner
How to use Jit-fnc/missionctrl_env with Docker Model Runner:
docker model run hf.co/Jit-fnc/missionctrl_env
File size: 3,063 Bytes
8723187 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Hugging Face Inference Endpoints custom handler for task=`custom`.
Loads Qwen2.5-0.5B-Instruct (or BASE_MODEL env) + LoRA adapter from the model repo.
Supports native `inputs` + `parameters` and optional `messages` payloads.
"""
from __future__ import annotations
import os
from typing import Any, Dict, List
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct")
class EndpointHandler:
"""Called by huggingface_inference_toolkit with model_dir as `path`."""
def __init__(self, path: str = "") -> None:
model_dir = path or os.environ.get("MODEL_DIR", ".")
self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
base = AutoModelForCausalLM.from_pretrained(
BASE_MODEL,
torch_dtype=torch.float16,
device_map="auto",
trust_remote_code=True,
)
self.model = PeftModel.from_pretrained(base, model_dir)
self.model.eval()
self._device = next(self.model.parameters()).device
def _prompt_from_messages(self, messages: List[Dict[str, Any]]) -> str:
return self.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
def __call__(self, data: Dict[str, Any]) -> List[Dict[str, str]]:
if "messages" in data:
prompt = self._prompt_from_messages(data["messages"])
params: Dict[str, Any] = {
k: data[k] for k in ("max_tokens", "temperature", "top_p") if k in data
}
if "max_tokens" in params:
params["max_new_tokens"] = int(params.pop("max_tokens"))
elif "max_new_tokens" not in params:
params["max_new_tokens"] = 120
else:
prompt = str(data.get("inputs") or "")
params = dict(data.get("parameters") or {})
return_full = bool(params.pop("return_full_text", False))
max_new = int(params.get("max_new_tokens", 120))
temperature = float(params.get("temperature", 0.0))
gen_kwargs: Dict[str, Any] = {
"max_new_tokens": max_new,
"temperature": temperature,
"do_sample": temperature > 0.0,
"pad_token_id": self.tokenizer.eos_token_id,
}
if "top_p" in params:
gen_kwargs["top_p"] = float(params["top_p"])
enc = self.tokenizer(prompt, return_tensors="pt")
enc = {k: v.to(self._device) for k, v in enc.items()}
with torch.no_grad():
out = self.model.generate(**enc, **gen_kwargs)
full_text = self.tokenizer.decode(out[0], skip_special_tokens=True)
prompt_text = self.tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True)
if return_full:
generated = full_text
else:
generated = full_text[len(prompt_text) :].strip()
return [{"generated_text": generated}]
|