Update app.py

app.py

@@ -1,179 +1,179 @@
-import spaces
-import json
-import subprocess
-from llama_cpp import Llama
-from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
-from llama_cpp_agent.providers import LlamaCppPythonProvider
-import gradio as gr
-from huggingface_hub import hf_hub_download
-import logging
-import time
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-repo_id = "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF"
-filename = "Meta-Llama-3-8B-Instruct.Q8_0.gguf"
-
-def chunk_text(text, chunk_size=5000):
-    """
-    Splits the input text into chunks of specified size.
-
-    Args:
-        text (str): The input text to be chunked.
-        chunk_size (int): The size of each chunk in tokens.
-
-    Returns:
-        list: A list of text chunks.
-    """
-    words = text.split()
-    chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
-    return chunks
-
-
-try:
-        start_time = time.time()
-        logger.info("Downloading Model....")
-
-        hf_hub_download(
-            repo_id = repo_id ,
-            filename = filename,
-            local_dir="./model"
-        )
-
-        end_time = time.time()
-        logger.info(f"Download complete. Time taken : {end_time - start_time} seconds.")
-
-except Exception as e:
-    logger.error(f"Unable to download Model : {e}")
-    raise
-
-llm = None
-
-@spaces.GPU(duration=120)
-def respond(message, history, max_tokens, temperature):
-    """
-    Generate a streaming response using the llama3-8b model with chunking.
-    
-    Args:
-        message (str): The input message.
-        history (list): The conversation history used by ChatInterface. - Not used.
-        temperature (float): The temperature for generating the response.
-        max_new_tokens (int): The maximum number of new tokens to generate.
-        
-    Returns:
-        str: The generated response.
-    """
-
-    chat_template = MessagesFormatterType.LLAMA_3
-
-    global llm
-    
-    start_time = time.time()
-    logging.info("Loading Model...")
-
-    if llm is None:
-        model = Llama(
-            model_path=f"models/{filename}",
-            flash_attn=True,
-            n_gpu_layers=-1,
-            n_batch=1,
-            n_ctx=8192,
-            last_n_tokens = 0
-        )
-        llm = model
-
-    end_time = time.time()
-    logger.info(f"Model Loaded. Time taken : {end_time - start_time} seconds.")
-
-    start_time = time.time()
-    logger.info("Loading Provider and Agent for the Llama Model....")
-
-    provider = LlamaCppPythonProvider(llm)
-
-    SYS_PROMPT ="""
-        Extract all relevant keywords and add quantity from the following text and format the result in nested JSON, ignoring personal details and focusing only on the scope of work as shown in the example:
-        Good JSON example: {'lobby': {'frcm': {'replace': {'carpet': 1, 'carpet_pad': 1, 'base': 1, 'window_treatments': 1, 'artwork_and_decorative_accessories': 1, 'portable_lighting': 1, 'upholstered_furniture_and_decorative_pillows': 1, 'millwork': 1} } } }
-        Bad JSON example: {'lobby': { 'frcm': { 'replace': [ 'carpet', 'carpet_pad', 'base', 'window_treatments', 'artwork_and_decorative_accessories', 'portable_lighting', 'upholstered_furniture_and_decorative_pillows', 'millwork'] } } }
-        Make sure to fetch details from the provided text and ignore unnecessary information. The response should be in JSON format only, without any additional comments.
-    """
-
-    agent = LlamaCppAgent(
-        provider,
-        system_prompt=SYS_PROMPT,
-        predefined_messages_formatter_type=chat_template,
-        debug_output=True
-    )
-    
-    settings = provider.get_provider_default_settings()
-    settings.temperature = temperature
-    settings.max_tokens = max_tokens
-    settings.stream = True
-
-    end_time = time.time()
-    logger.info(f"Provider settings updated. Prompt Loaded.Time taken : {end_time - start_time} seconds.")
-    
-    chunks = chunk_text(message)
-
-    start_time = time.time()
-    logger.info("Generating responses...")
-    for chunk in chunks:
-        response = agent.get_chat_response(
-            chunk,
-            llm_sampling_settings=settings,
-            returns_streaming_generator = False, #generate streamer
-            print_output = False
-        )
-    logger.info(f"Responses generated. Time taken : {time.time() - start_time} seconds.")
-
-    return response
-
-DESCRIPTION = '''
-<div>
-<h1 style="text-align: center;">ContenteaseAI custom trained model</h1>
-</div>
-'''
-
-LICENSE = """
-<p/>
----
-For more information, visit our [website](https://contentease.ai).
-"""
-
-PLACEHOLDER = """
-<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
-   <h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">ContenteaseAI Custom AI trained model</h1>
-   <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Enter the text extracted from the PDF:</p>
-</div>
-"""
-
-css = """
-h1 {
-  text-align: center;
-  display: block;
-}
-"""
-# Gradio block
-chatbot = gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface')
-
-with gr.Blocks(fill_height=True, css=css) as demo:
-    gr.Markdown(DESCRIPTION)
-
-    gr.ChatInterface(
-        fn=respond,
-        chatbot=chatbot,
-        fill_height=True,
-        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
-        additional_inputs=[
-            gr.Slider(minimum=0, maximum=1, step=0.1, value=0.90, label="Temperature", render=False),
-            gr.Slider(minimum=128, maximum=2000, step=1, value=1500, label="Max new tokens", render=False),
-        ]
-    )
-
-    gr.Markdown(LICENSE)
-
-if __name__ == "__main__":
-    try:
-        demo.launch(show_error=True, debug = True)
-    except Exception as e:
+import spaces
+import json
+import subprocess
+from llama_cpp import Llama
+from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
+from llama_cpp_agent.providers import LlamaCppPythonProvider
+import gradio as gr
+from huggingface_hub import hf_hub_download
+import logging
+import time
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+repo_id = "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF"
+filename = "Meta-Llama-3-8B-Instruct.Q8_0.gguf"
+
+def chunk_text(text, chunk_size=5000):
+    """
+    Splits the input text into chunks of specified size.
+
+    Args:
+        text (str): The input text to be chunked.
+        chunk_size (int): The size of each chunk in tokens.
+
+    Returns:
+        list: A list of text chunks.
+    """
+    words = text.split()
+    chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+    return chunks
+
+
+try:
+        start_time = time.time()
+        logger.info("Downloading Model....")
+
+        hf_hub_download(
+            repo_id = repo_id ,
+            filename = filename,
+            local_dir="./model"
+        )
+
+        end_time = time.time()
+        logger.info(f"Download complete. Time taken : {end_time - start_time} seconds.")
+
+except Exception as e:
+    logger.error(f"Unable to download Model : {e}")
+    raise
+
+llm = None
+
+@spaces.GPU(duration=120)
+def respond(message, history, max_tokens, temperature):
+    """
+    Generate a streaming response using the llama3-8b model with chunking.
+    
+    Args:
+        message (str): The input message.
+        history (list): The conversation history used by ChatInterface. - Not used.
+        temperature (float): The temperature for generating the response.
+        max_new_tokens (int): The maximum number of new tokens to generate.
+        
+    Returns:
+        str: The generated response.
+    """
+
+    chat_template = MessagesFormatterType.LLAMA_3
+
+    global llm
+    
+    start_time = time.time()
+    logging.info("Loading Model...")
+
+    if llm is None:
+        model = Llama(
+            model_path=f"model/{filename}",
+            flash_attn=True,
+            n_gpu_layers=-1,
+            n_batch=1,
+            n_ctx=8192,
+            last_n_tokens = 0
+        )
+        llm = model
+
+    end_time = time.time()
+    logger.info(f"Model Loaded. Time taken : {end_time - start_time} seconds.")
+
+    start_time = time.time()
+    logger.info("Loading Provider and Agent for the Llama Model....")
+
+    provider = LlamaCppPythonProvider(llm)
+
+    SYS_PROMPT ="""
+        Extract all relevant keywords and add quantity from the following text and format the result in nested JSON, ignoring personal details and focusing only on the scope of work as shown in the example:
+        Good JSON example: {'lobby': {'frcm': {'replace': {'carpet': 1, 'carpet_pad': 1, 'base': 1, 'window_treatments': 1, 'artwork_and_decorative_accessories': 1, 'portable_lighting': 1, 'upholstered_furniture_and_decorative_pillows': 1, 'millwork': 1} } } }
+        Bad JSON example: {'lobby': { 'frcm': { 'replace': [ 'carpet', 'carpet_pad', 'base', 'window_treatments', 'artwork_and_decorative_accessories', 'portable_lighting', 'upholstered_furniture_and_decorative_pillows', 'millwork'] } } }
+        Make sure to fetch details from the provided text and ignore unnecessary information. The response should be in JSON format only, without any additional comments.
+    """
+
+    agent = LlamaCppAgent(
+        provider,
+        system_prompt=SYS_PROMPT,
+        predefined_messages_formatter_type=chat_template,
+        debug_output=True
+    )
+    
+    settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.max_tokens = max_tokens
+    settings.stream = True
+
+    end_time = time.time()
+    logger.info(f"Provider settings updated. Prompt Loaded.Time taken : {end_time - start_time} seconds.")
+    
+    chunks = chunk_text(message)
+
+    start_time = time.time()
+    logger.info("Generating responses...")
+    for chunk in chunks:
+        response = agent.get_chat_response(
+            chunk,
+            llm_sampling_settings=settings,
+            returns_streaming_generator = False, #generate streamer
+            print_output = False
+        )
+    logger.info(f"Responses generated. Time taken : {time.time() - start_time} seconds.")
+
+    return response
+
+DESCRIPTION = '''
+<div>
+<h1 style="text-align: center;">ContenteaseAI custom trained model</h1>
+</div>
+'''
+
+LICENSE = """
+<p/>
+---
+For more information, visit our [website](https://contentease.ai).
+"""
+
+PLACEHOLDER = """
+<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
+   <h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">ContenteaseAI Custom AI trained model</h1>
+   <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Enter the text extracted from the PDF:</p>
+</div>
+"""
+
+css = """
+h1 {
+  text-align: center;
+  display: block;
+}
+"""
+# Gradio block
+chatbot = gr.Chatbot(height=450, placeholder=PLACEHOLDER, label='Gradio ChatInterface')
+
+with gr.Blocks(fill_height=True, css=css) as demo:
+    gr.Markdown(DESCRIPTION)
+
+    gr.ChatInterface(
+        fn=respond,
+        chatbot=chatbot,
+        fill_height=True,
+        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
+        additional_inputs=[
+            gr.Slider(minimum=0, maximum=1, step=0.1, value=0.90, label="Temperature", render=False),
+            gr.Slider(minimum=128, maximum=2000, step=1, value=1500, label="Max new tokens", render=False),
+        ]
+    )
+
+    gr.Markdown(LICENSE)
+
+if __name__ == "__main__":
+    try:
+        demo.launch(show_error=True, debug = True)
+    except Exception as e:
         logger.error(f"Error launching Gradio demo: {e}")