new app.py

.ipynb_checkpoints/app-checkpoint.py

@@ -2,7 +2,7 @@ import gradio as gr
 import tqdm
 import torch
 from torch import nn
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 from functools import partial
 import gc
 
@@ -62,9 +62,18 @@ offload_folder = "offload"
 tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
 model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", offload_folder=offload_folder)
 
-model_q = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto",, offload_folder=offload_folder)
+model_q = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto",offload_folder=offload_folder)
 pseudo_quantize_model_weight(model_q, w_bit=3, q_group_size=128)
 # Define a function for model inference
+
+
+generator = pipeline('text-generation', model="facebook/opt-1.3b")
+
+def generate_text_pip(prompt):
+    generated_text = generator(prompt, max_length=1000, num_return_sequences=1)[0]['generated_text']
+    return generated_text
+print(generator("I went to boston and"))
+
 def generate_text(prompt):
     inputs = tokenizer(prompt, return_tensors="pt")
     output = model(**inputs)
@@ -82,7 +91,7 @@ def generate_text_from_quantized(prompt):
     return generated_text
 
 # Create a Gradio interface
-iface = gr.Interface(fn=generate_text, inputs="text", outputs="text", live=True)
+iface = gr.Interface(fn=generate_text_pip, inputs="text", outputs="text", live=True)
 
 iface_2 = gr.Interface(fn=generate_text_from_quantized, inputs="text", outputs="text", live=True)
 
@@ -90,4 +99,4 @@ iface_2 = gr.Interface(fn=generate_text_from_quantized, inputs="text", outputs="
 app = gr.TabbedInterface([iface, iface_2],["Normal", "Quantized"])
 
 # Launch the Gradio app
-app.launch(share=True)
+app.launch(server_name="0.0.0.0", share=True)

app.py

@@ -59,18 +59,13 @@ def pseudo_quantize_model_weight(
             
 # Load the tokenizer and model
 model_path = "facebook/opt-125m"
-offload_folder = "offload"
-tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
-model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", offload_folder=offload_folder)
-model.to('cpu')
 
 model_q_path = "facebook/opt-125m_3bit"
-model_q = AutoModelForCausalLM.from_pretrained(model_q_path, device_map="auto", offload_folder=offload_folder)
-model_q.to('cpu')
-#pseudo_quantize_model_weight(model_q, w_bit=3, q_group_size=128)
+
 
 generator = pipeline('text-generation', model="facebook/opt-125m")
-generator_q = pipeline('text-generation', model=model_q)
+generator_q = pipeline('text-generation', model="facebook/opt-125m_3bit")
+
 
 def generate_text_pip(prompt):
     generated_text = generator(prompt, max_length=250, num_return_sequences=1)[0]['generated_text']
@@ -80,7 +75,10 @@ def generate_text_pip_q(prompt):
     generated_text = generator_q(prompt, max_length=250, num_return_sequences=1)[0]['generated_text']
     return generated_text
 
-# Define a function for model inference
+print(generator("I went to boston and"))
+print("quantized model",generator_q("I went to boston and"))
+
+'''
 def generate_text(prompt):
     inputs = tokenizer(prompt, return_tensors="pt")
     output = model(**inputs)
@@ -98,7 +96,7 @@ def generate_text_from_quantized(prompt):
     #generated_text = tokenizer.decode(predicted_ids[0], skip_special_tokens=True)
     generated_text = tokenizer.batch_decode(predicted_ids, skip_special_tokens=True)[0]
     return generated_text
-
+'''
 # Create a Gradio interface
 iface = gr.Interface(fn=generate_text_pip, inputs="text", outputs="text")
 
@@ -108,4 +106,4 @@ iface_2 = gr.Interface(fn=generate_text_pip_q, inputs="text", outputs="text")
 app = gr.TabbedInterface([iface, iface_2],["Normal", "Quantized"])
 
 # Launch the Gradio app
-app.launch()
+app.launch()