Spaces:
Runtime error
Runtime error
File size: 1,119 Bytes
fd826ae 71fbc88 fd826ae 71fbc88 fd826ae 71fbc88 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | import gradio as gr
import numpy as np
import torch
from transformers import pipeline
import torch
import os
# you only need a huggingface API token when accessing gated models like llama
hf_token = os.getenv("HF_TOKEN")
# Pick any model from huggingface hub that supports text completion
model_id = "HuggingFaceH4/zephyr-7b-alpha"
# Load the model using the transformers pipeline
model_pipeline = pipeline(
"text-generation",
model=model_id, # model_id can be any huggingface model
device_map="auto", # Uses the GPU if available
torch_dtype=torch.bfloat16, # Saves you memory
token=hf_token, # Necessary for gated models like llama2
)
def inference(text: str, max_length: int=100) -> str:
return model_pipeline(text, max_length=max_length)[0]["generated_text"]
# Setup the Gradio Interface
title = f"Inference for {model_id}"
description = f"This demo uses {model_id} for inference"
io = gr.Interface(
inference,
inputs=gr.Textbox(lines=3, label="Model Input"),
outputs=gr.Textbox(lines=3, label="Model's Output"),
title=title,
description=description,
)
io.launch(share=True)
|