Instructions to use lucafirefox/Phi-3-small-8k-instruct_handler with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use lucafirefox/Phi-3-small-8k-instruct_handler with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="lucafirefox/Phi-3-small-8k-instruct_handler", trust_remote_code=True)
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("lucafirefox/Phi-3-small-8k-instruct_handler", trust_remote_code=True, device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use lucafirefox/Phi-3-small-8k-instruct_handler with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "lucafirefox/Phi-3-small-8k-instruct_handler"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lucafirefox/Phi-3-small-8k-instruct_handler",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/lucafirefox/Phi-3-small-8k-instruct_handler

SGLang

How to use lucafirefox/Phi-3-small-8k-instruct_handler with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "lucafirefox/Phi-3-small-8k-instruct_handler" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lucafirefox/Phi-3-small-8k-instruct_handler",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "lucafirefox/Phi-3-small-8k-instruct_handler" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "lucafirefox/Phi-3-small-8k-instruct_handler",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use lucafirefox/Phi-3-small-8k-instruct_handler with Docker Model Runner:
```
docker model run hf.co/lucafirefox/Phi-3-small-8k-instruct_handler
```

Phi-3-small-8k-instruct_handler / handler.py

lucafirefox

postprocess

94bd5b3 over 2 years ago

Raw History Blame Contribute Delete

1.64 kB

import torch
import subprocess
import sys
from typing import Dict, List, Any
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline

class EndpointHandler:
    def __init__(self, path=""):
        
        subprocess.check_call([sys.executable, "-m", "pip", "install", "flash-attn"])
        subprocess.check_call([sys.executable, "-m", "pip", "install", "tiktoken"])
        subprocess.check_call([sys.executable, "-m", "pip", "install", "pytest"])
        
        # load the model
        tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
        model = AutoModelForCausalLM.from_pretrained(path, device_map="auto",torch_dtype="auto", trust_remote_code=True)
        
        # create inference pipeline
        self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=4096)

    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
        inputs = data.pop("inputs", data)
        parameters = data.pop("parameters", None)

        messages = [
            {"role": "system", "content": "Translate the text to English. Rephrase if needed to ensure it sounds natural. Output only the translated text."},
            {"role": "user", "content": inputs},
        ]
        
        # pass inputs with all kwargs in data
        if parameters is not None:
            prediction = self.pipeline(messages, **parameters)
        else:
            prediction = self.pipeline(messages)
            
        # postprocess the prediction
        assistant_output = prediction[0]["generated_text"][-1]
        
        return {"output": assistant_output["content"]}

	import torch
	import subprocess
	import sys
	from typing import Dict, List, Any
	from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline

	class EndpointHandler:
	def __init__(self, path=""):

	subprocess.check_call([sys.executable, "-m", "pip", "install", "flash-attn"])
	subprocess.check_call([sys.executable, "-m", "pip", "install", "tiktoken"])
	subprocess.check_call([sys.executable, "-m", "pip", "install", "pytest"])

	# load the model
	tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
	model = AutoModelForCausalLM.from_pretrained(path, device_map="auto",torch_dtype="auto", trust_remote_code=True)

	# create inference pipeline
	self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=4096)

	def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
	inputs = data.pop("inputs", data)
	parameters = data.pop("parameters", None)

	messages = [
	{"role": "system", "content": "Translate the text to English. Rephrase if needed to ensure it sounds natural. Output only the translated text."},
	{"role": "user", "content": inputs},
	]

	# pass inputs with all kwargs in data
	if parameters is not None:
	prediction = self.pipeline(messages, **parameters)
	else:
	prediction = self.pipeline(messages)

	# postprocess the prediction
	assistant_output = prediction[0]["generated_text"][-1]

	return {"output": assistant_output["content"]}