File size: 1,119 Bytes
fd826ae
71fbc88
 
 
 
 
fd826ae
71fbc88
 
fd826ae
71fbc88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import gradio as gr
import numpy as np
import torch
from transformers import pipeline
import torch
import os

# you only need a huggingface API token when accessing gated models like llama
hf_token = os.getenv("HF_TOKEN")

# Pick any model from huggingface hub that supports text completion
model_id = "HuggingFaceH4/zephyr-7b-alpha"

# Load the model using the transformers pipeline
model_pipeline = pipeline(
    "text-generation",
    model=model_id,  # model_id can be any huggingface model
    device_map="auto",  # Uses the GPU if available
    torch_dtype=torch.bfloat16,  # Saves you memory
    token=hf_token, # Necessary for gated models like llama2
)

def inference(text: str, max_length: int=100) -> str:
  return model_pipeline(text, max_length=max_length)[0]["generated_text"]

# Setup the Gradio Interface
title = f"Inference for {model_id}"
description = f"This demo uses {model_id} for inference"

io = gr.Interface(
  inference,
  inputs=gr.Textbox(lines=3, label="Model Input"),
  outputs=gr.Textbox(lines=3, label="Model's Output"),
  title=title,
  description=description,
)
io.launch(share=True)