import gradio as gr import numpy as np import torch from transformers import pipeline import torch import os # you only need a huggingface API token when accessing gated models like llama hf_token = os.getenv("HF_TOKEN") # Pick any model from huggingface hub that supports text completion model_id = "HuggingFaceH4/zephyr-7b-alpha" # Load the model using the transformers pipeline model_pipeline = pipeline( "text-generation", model=model_id, # model_id can be any huggingface model device_map="auto", # Uses the GPU if available torch_dtype=torch.bfloat16, # Saves you memory token=hf_token, # Necessary for gated models like llama2 ) def inference(text: str, max_length: int=100) -> str: return model_pipeline(text, max_length=max_length)[0]["generated_text"] # Setup the Gradio Interface title = f"Inference for {model_id}" description = f"This demo uses {model_id} for inference" io = gr.Interface( inference, inputs=gr.Textbox(lines=3, label="Model Input"), outputs=gr.Textbox(lines=3, label="Model's Output"), title=title, description=description, ) io.launch(share=True)