test2 / app.py
Cyboman1337's picture
2
71fbc88
Raw History Blame Contribute Delete
1.12 kB
import gradio as gr
import numpy as np
import torch
from transformers import pipeline
import torch
import os
# you only need a huggingface API token when accessing gated models like llama
hf_token = os.getenv("HF_TOKEN")
# Pick any model from huggingface hub that supports text completion
model_id = "HuggingFaceH4/zephyr-7b-alpha"
# Load the model using the transformers pipeline
model_pipeline = pipeline(
"text-generation",
model=model_id, # model_id can be any huggingface model
device_map="auto", # Uses the GPU if available
torch_dtype=torch.bfloat16, # Saves you memory
token=hf_token, # Necessary for gated models like llama2
)
def inference(text: str, max_length: int=100) -> str:
return model_pipeline(text, max_length=max_length)[0]["generated_text"]
# Setup the Gradio Interface
title = f"Inference for {model_id}"
description = f"This demo uses {model_id} for inference"
io = gr.Interface(
inference,
inputs=gr.Textbox(lines=3, label="Model Input"),
outputs=gr.Textbox(lines=3, label="Model's Output"),
title=title,
description=description,
)
io.launch(share=True)