KajetanNiewczas
Bugs, bugs, bugs
b554ea9
Raw History Blame Contribute Delete
2.64 kB
import os
import gradio as gr
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
from peft import PeftModel
import torch
hf_token = os.environ.get("HF_TOKEN")
device = "cuda" if torch.cuda.is_available() else "cpu"
base_model_name = "allenai/led-base-16384"
peft_model_name = "manchot71/led-app-5120-lora"
# Load model and tokenizer
tokenizer = AutoTokenizer.from_pretrained(base_model_name)
base_model = AutoModelForSeq2SeqLM.from_pretrained(base_model_name)
model = PeftModel.from_pretrained(base_model, peft_model_name, token=hf_token)
model = model.to(device)
MAX_LENGTH = 5120
def truncate_front_back(text, tokenizer, max_length=MAX_LENGTH):
tokenized = tokenizer(text, add_special_tokens=False)
input_ids = tokenized["input_ids"]
if len(input_ids) <= max_length:
return input_ids
half = max_length // 2
return input_ids[:half] + input_ids[-(max_length - half) :]
def infer(prompt, max_output_length, num_beams):
# Tokenize input
input_ids = truncate_front_back(prompt, tokenizer, max_length=MAX_LENGTH)
input_ids = torch.tensor([input_ids]).to(device)
# Generate output
outputs = model.generate(
input_ids=input_ids,
max_length=max_output_length,
num_beams=num_beams,
early_stopping=True,
)
# Decode generated text
decoded_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
return decoded_output
css = """
#col-container {
margin: 0 auto;
max-width: 640px;
}
"""
with gr.Blocks(css=css) as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown("# Abstract for your physics paper")
prompt = gr.Textbox(
label="Prompt",
show_label=False,
max_lines=5,
placeholder="Enter your prompt",
)
run_button = gr.Button("Run", scale=0, variant="primary")
result = gr.Textbox(label="Result", show_label=False, max_lines=10)
with gr.Accordion("Advanced Settings", open=False):
max_output_length = gr.Slider(
label="Max output length", minimum=10, maximum=2048, step=10, value=512
)
num_beams = gr.Slider(
label="Number of beams",
minimum=1,
maximum=10,
step=1,
value=3,
)
run_button.click(
fn=infer, inputs=[prompt, max_output_length, num_beams], outputs=[result]
)
prompt.submit(
fn=infer, inputs=[prompt, max_output_length, num_beams], outputs=[result]
)
if __name__ == "__main__":
demo.launch()