Text Generation
Transformers
Safetensors
PEFT
English
unsloth
qwen2
trl
lora
missionctrl
ai-oversight
conversational
Instructions to use Jit-fnc/missionctrl_env with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Jit-fnc/missionctrl_env with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="Jit-fnc/missionctrl_env") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Jit-fnc/missionctrl_env", device_map="auto") - PEFT
How to use Jit-fnc/missionctrl_env with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Jit-fnc/missionctrl_env with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Jit-fnc/missionctrl_env" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/Jit-fnc/missionctrl_env
- SGLang
How to use Jit-fnc/missionctrl_env with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Jit-fnc/missionctrl_env" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Jit-fnc/missionctrl_env" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Jit-fnc/missionctrl_env", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Unsloth Desktop
- Docker Model Runner
How to use Jit-fnc/missionctrl_env with Docker Model Runner:
docker model run hf.co/Jit-fnc/missionctrl_env
Download handler.py from Jit-fnc/missionctrl_env: direct link, hf CLI and curl.
- Browser
- Download file 3.06 kB
-
https://huggingface.co/Jit-fnc/missionctrl_env/resolve/main/handler.py
- Command line
-
hf download hf://Jit-fnc/missionctrl_env/handler.py
-
curl -L -o handler.py https://huggingface.co/Jit-fnc/missionctrl_env/resolve/main/handler.py
3.06 kB
| """Hugging Face Inference Endpoints custom handler for task=`custom`. | |
| Loads Qwen2.5-0.5B-Instruct (or BASE_MODEL env) + LoRA adapter from the model repo. | |
| Supports native `inputs` + `parameters` and optional `messages` payloads. | |
| """ | |
| from __future__ import annotations | |
| import os | |
| from typing import Any, Dict, List | |
| import torch | |
| from peft import PeftModel | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct") | |
| class EndpointHandler: | |
| """Called by huggingface_inference_toolkit with model_dir as `path`.""" | |
| def __init__(self, path: str = "") -> None: | |
| model_dir = path or os.environ.get("MODEL_DIR", ".") | |
| self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) | |
| base = AutoModelForCausalLM.from_pretrained( | |
| BASE_MODEL, | |
| torch_dtype=torch.float16, | |
| device_map="auto", | |
| trust_remote_code=True, | |
| ) | |
| self.model = PeftModel.from_pretrained(base, model_dir) | |
| self.model.eval() | |
| self._device = next(self.model.parameters()).device | |
| def _prompt_from_messages(self, messages: List[Dict[str, Any]]) -> str: | |
| return self.tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=False, | |
| add_generation_prompt=True, | |
| ) | |
| def __call__(self, data: Dict[str, Any]) -> List[Dict[str, str]]: | |
| if "messages" in data: | |
| prompt = self._prompt_from_messages(data["messages"]) | |
| params: Dict[str, Any] = { | |
| k: data[k] for k in ("max_tokens", "temperature", "top_p") if k in data | |
| } | |
| if "max_tokens" in params: | |
| params["max_new_tokens"] = int(params.pop("max_tokens")) | |
| elif "max_new_tokens" not in params: | |
| params["max_new_tokens"] = 120 | |
| else: | |
| prompt = str(data.get("inputs") or "") | |
| params = dict(data.get("parameters") or {}) | |
| return_full = bool(params.pop("return_full_text", False)) | |
| max_new = int(params.get("max_new_tokens", 120)) | |
| temperature = float(params.get("temperature", 0.0)) | |
| gen_kwargs: Dict[str, Any] = { | |
| "max_new_tokens": max_new, | |
| "temperature": temperature, | |
| "do_sample": temperature > 0.0, | |
| "pad_token_id": self.tokenizer.eos_token_id, | |
| } | |
| if "top_p" in params: | |
| gen_kwargs["top_p"] = float(params["top_p"]) | |
| enc = self.tokenizer(prompt, return_tensors="pt") | |
| enc = {k: v.to(self._device) for k, v in enc.items()} | |
| with torch.no_grad(): | |
| out = self.model.generate(**enc, **gen_kwargs) | |
| full_text = self.tokenizer.decode(out[0], skip_special_tokens=True) | |
| prompt_text = self.tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True) | |
| if return_full: | |
| generated = full_text | |
| else: | |
| generated = full_text[len(prompt_text) :].strip() | |
| return [{"generated_text": generated}] | |