import os import gradio as gr from transformers import AutoTokenizer, AutoModelForSeq2SeqLM from peft import PeftModel import torch hf_token = os.environ.get("HF_TOKEN") device = "cuda" if torch.cuda.is_available() else "cpu" base_model_name = "allenai/led-base-16384" peft_model_name = "manchot71/led-app-5120-lora" # Load model and tokenizer tokenizer = AutoTokenizer.from_pretrained(base_model_name) base_model = AutoModelForSeq2SeqLM.from_pretrained(base_model_name) model = PeftModel.from_pretrained(base_model, peft_model_name, token=hf_token) model = model.to(device) MAX_LENGTH = 5120 def truncate_front_back(text, tokenizer, max_length=MAX_LENGTH): tokenized = tokenizer(text, add_special_tokens=False) input_ids = tokenized["input_ids"] if len(input_ids) <= max_length: return input_ids half = max_length // 2 return input_ids[:half] + input_ids[-(max_length - half) :] def infer(prompt, max_output_length, num_beams): # Tokenize input input_ids = truncate_front_back(prompt, tokenizer, max_length=MAX_LENGTH) input_ids = torch.tensor([input_ids]).to(device) # Generate output outputs = model.generate( input_ids=input_ids, max_length=max_output_length, num_beams=num_beams, early_stopping=True, ) # Decode generated text decoded_output = tokenizer.decode(outputs[0], skip_special_tokens=True) return decoded_output css = """ #col-container { margin: 0 auto; max-width: 640px; } """ with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.Markdown("# Abstract for your physics paper") prompt = gr.Textbox( label="Prompt", show_label=False, max_lines=5, placeholder="Enter your prompt", ) run_button = gr.Button("Run", scale=0, variant="primary") result = gr.Textbox(label="Result", show_label=False, max_lines=10) with gr.Accordion("Advanced Settings", open=False): max_output_length = gr.Slider( label="Max output length", minimum=10, maximum=2048, step=10, value=512 ) num_beams = gr.Slider( label="Number of beams", minimum=1, maximum=10, step=1, value=3, ) run_button.click( fn=infer, inputs=[prompt, max_output_length, num_beams], outputs=[result] ) prompt.submit( fn=infer, inputs=[prompt, max_output_length, num_beams], outputs=[result] ) if __name__ == "__main__": demo.launch()