Makhinur commited on
Commit
a56d7be
·
verified ·
1 Parent(s): b1abf7a

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +43 -40
main.py CHANGED
@@ -5,14 +5,15 @@ from fastapi import FastAPI, File, UploadFile, Form, HTTPException
5
  # from fastapi.templating import Jinja2Templates
6
  # from fastapi.responses import FileResponse
7
 
8
- # Removed 'requests' as we are using gradio_client
9
  # import requests
10
  import base64 # Keep if needed elsewhere (not strictly needed for this version)
11
  import os
12
  import random
13
- from typing import IO # For type hinting file-like objects
 
14
 
15
- # Import necessary classes from transformers (Keeping only AutoTokenizer for chat templating)
16
  from transformers import AutoTokenizer
17
 
18
  # Import necessary modules for llama-cpp-python and downloading from Hub
@@ -36,10 +37,14 @@ from deep_translator.exceptions import InvalidSourceOrTargetLanguage
36
  app = FastAPI()
37
 
38
  # --- Llama.cpp Language Model Setup (Local CPU Inference) ---
39
- # Repository on Hugging Face Hub containing the GGUF file
40
- # Choose a specific GGUF file from the repo. Q4_K_M is a good balance for speed/quality on CPU.
41
- LLM_MODEL_REPO = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
42
- LLM_MODEL_FILE = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf" # Example GGUF file name
 
 
 
 
43
 
44
  tokenizer = None # Using transformers tokenizer for chat templating
45
  llm_model = None # This will hold the llama_cpp.Llama instance
@@ -58,12 +63,9 @@ def load_language_model():
58
  print(f"Loading language model: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
59
  try:
60
  # --- Load Tokenizer (using transformers) ---
61
- # We load the tokenizer from the original model repo (not the GGUF specific one)
62
- # The tokenizer is needed for applying chat templates correctly.
63
- # Extract the base model name from the GGUF repo name
64
- original_model_name = LLM_MODEL_REPO.split('/')[1].replace('-GGUF', '')
65
- print(f"Loading tokenizer from original model repo: TinyLlama/{original_model_name}...")
66
- tokenizer = AutoTokenizer.from_pretrained(f"TinyLlama/{original_model_name}")
67
 
68
  # Set pad_token if not already defined, often necessary for correct batching/generation behavior
69
  if tokenizer.pad_token is None:
@@ -79,11 +81,10 @@ def load_language_model():
79
 
80
  # --- Download GGUF model file (using huggingface_hub) ---
81
  print(f"Downloading GGUF model file: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
82
- # hf_hub_download downloads the file to the Hugging Face cache directory
83
  model_path = hf_hub_download(
84
  repo_id=LLM_MODEL_REPO,
85
  filename=LLM_MODEL_FILE,
86
- # cache_dir="/tmp/hf_cache" # Optional: specify a custom cache directory if needed
87
  )
88
  print(f"GGUF model downloaded to: {model_path}")
89
 
@@ -91,13 +92,13 @@ def load_language_model():
91
  print(f"Loading GGUF model into llama_cpp...")
92
  # Instantiate the Llama model from the downloaded GGUF file
93
  # n_gpu_layers=0: Crucial for forcing CPU-only inference
94
- # n_ctx: Context window size (tokens model can consider), match model's spec if possible
95
  # n_threads: Number of CPU threads to use. Set to your vCPU count (2) for better performance.
96
  llm_model = Llama(
97
  model_path=model_path,
98
  n_gpu_layers=0, # Explicitly use CPU
99
- n_ctx=4096, # Context window size (check model card if unsure, 4096 is common)
100
- n_threads=2 # Use 2 CPU threads on your basic tier
101
  )
102
  print("Llama.cpp model loaded successfully.")
103
 
@@ -112,9 +113,8 @@ def initialize_caption_client():
112
  global caption_client
113
  print(f"Initializing Gradio client for {CAPTION_SPACE_URL}...")
114
  try:
115
- # If the target Gradio Space requires authentication (e.g., if it's private)
116
- # You'll need to pass your Hugging Face User Access Token.
117
- # Store this token as a Space Secret named HF_TOKEN.
118
  # HF_TOKEN = os.environ.get("HF_TOKEN")
119
  # if HF_TOKEN:
120
  # print("Using HF_TOKEN for Gradio client.")
@@ -123,7 +123,7 @@ def initialize_caption_client():
123
  # print("HF_TOKEN not found. Initializing public Gradio client.")
124
  # caption_client = Client(CAPTION_SPACE_URL)
125
 
126
- # Assuming the caption space is public and does not require a token
127
  caption_client = Client(CAPTION_SPACE_URL)
128
  print("Gradio client initialized successfully.")
129
  except Exception as e:
@@ -135,7 +135,7 @@ def initialize_caption_client():
135
  # Load models and initialize clients when the app starts
136
  @app.on_event("startup")
137
  async def startup_event():
138
- # Load the language model (TinyLlama GGUF via llama.cpp)
139
  load_language_model()
140
  # Initialize the client for the external captioning Space
141
  initialize_caption_client()
@@ -202,29 +202,32 @@ def generate_image_caption(image_file: UploadFile):
202
  print(f"Error removing temporary file {temp_file_path}: {e}") # Log cleanup errors
203
 
204
 
205
- # --- Language Model Story Generation Function (TinyLlama via llama.cpp) ---
206
- def generate_story_tinyllama(prompt_text: str, max_new_tokens: int = 300, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 50) -> str:
 
207
  """
208
- Generates text using the loaded TinyLlama model via llama.cpp.
209
  Uses the tokenizer to apply the chat template and calls llama.cpp's chat completion.
210
  """
211
  # Check if the language model was loaded successfully at startup
 
212
  if tokenizer is None or llm_model is None:
213
  # Raise a RuntimeError which is caught by the calling endpoint
214
  raise RuntimeError("Language model (llama.cpp) or tokenizer not loaded.")
215
 
216
- # Construct the messages list following the chat format (e.g., for TinyLlama Chat)
 
217
  messages = [
 
 
218
  {"role": "user", "content": prompt_text}
219
- # You could add a system message here if the model was trained with one
220
- # e.g., {"role": "system", "content": "You are a creative storyteller."}
221
  ]
222
 
223
  try:
224
- print("Calling llama.cpp create_chat_completion...")
225
  # Call the create_chat_completion method from llama_cpp.Llama instance
226
- # This method handles the chat templating internally for many models.
227
- # max_tokens is the max number of tokens to generate (equivalent to max_new_tokens)
228
  # temperature, top_p control sampling. top_k might not be a direct parameter.
229
  response = llm_model.create_chat_completion(
230
  messages=messages,
@@ -233,9 +236,9 @@ def generate_story_tinyllama(prompt_text: str, max_new_tokens: int = 300, temper
233
  top_p=top_p,
234
  # top_k is sometimes supported, but check llama-cpp-python docs if needed
235
  # top_k=top_k,
236
- stream=False # We want the full response at once, not streaming
237
  )
238
- print("Llama.cpp completion received.")
239
 
240
  # Parse the response to get the generated text content
241
  # The response structure is typically like OpenAI's chat API response
@@ -243,12 +246,12 @@ def generate_story_tinyllama(prompt_text: str, max_new_tokens: int = 300, temper
243
  story = response['choices'][0].get('message', {}).get('content', '')
244
  else:
245
  # Handle cases where the response is empty or has an unexpected structure
246
- print("Warning: Llama.cpp response structure unexpected or content missing.")
247
  story = "" # Return an empty string if content is not found
248
 
249
  except Exception as e:
250
  # Catch any exception that occurs during the llama.cpp inference process
251
- print(f"Llama.cpp inference failed: {e}") # Log the error server-side
252
  # Re-raise as a RuntimeError to indicate failure to the endpoint
253
  raise RuntimeError(f"Llama.cpp inference failed: {type(e).__name__}: {e}")
254
 
@@ -282,10 +285,10 @@ async def generate_story_endpoint(image_file: UploadFile = File(...), language:
282
  # This prompt instructs the model on what to write and incorporates the caption.
283
  prompt_text = f"Write an attractive story of around 300 words about {story_theme}. Incorporate the following details from an image description into the story: {caption}\n\nStory:"
284
 
285
- # Step 3: Generate the story using the local language model (TinyLlama via llama.cpp)
286
  try:
287
- # Call the story generation function
288
- story = generate_story_tinyllama(
289
  prompt_text,
290
  max_new_tokens=300, # Request ~300 new tokens
291
  temperature=0.7, # Sampling parameters
@@ -295,7 +298,7 @@ async def generate_story_endpoint(image_file: UploadFile = File(...), language:
295
  story = story.strip() # Basic cleanup of generated story text
296
 
297
  except RuntimeError as e:
298
- # Catch specific RuntimeError raised by generate_story_tinyllama if LLM loading or inference fails
299
  print(f"Language model generation error: {e}") # Log the error server-side
300
  # Return a 503 Service Unavailable error if the LLM is not available or failed
301
  raise HTTPException(status_code=503, detail=f"Story generation failed (LLM): {e}")
 
5
  # from fastapi.templating import Jinja2Templates
6
  # from fastapi.responses import FileResponse
7
 
8
+ # Removed 'requests'
9
  # import requests
10
  import base64 # Keep if needed elsewhere (not strictly needed for this version)
11
  import os
12
  import random
13
+ # Removed unused IO import
14
+ # from typing import IO
15
 
16
+ # Import necessary classes from transformers (Keeping only AutoTokenizer)
17
  from transformers import AutoTokenizer
18
 
19
  # Import necessary modules for llama-cpp-python and downloading from Hub
 
37
  app = FastAPI()
38
 
39
  # --- Llama.cpp Language Model Setup (Local CPU Inference) ---
40
+ # Repository on Hugging Face Hub containing the Qwen1.5 1.8B GGUF file
41
+ # Using TheBloke's repo as a common source for GGUF models.
42
+ LLM_MODEL_REPO = "TheBloke/Qwen1.5-1.8B-Chat-GGUF"
43
+ # Specify the filename for a Q4_K_M quantized version (good balance of speed/quality on CPU)
44
+ LLM_MODEL_FILE = "qwen1_5-1_8b-chat-Q4_K_M.gguf"
45
+
46
+ # Original model name for the tokenizer (needed by transformers)
47
+ ORIGINAL_MODEL_NAME = "Qwen/Qwen1.5-1.8B-Chat"
48
 
49
  tokenizer = None # Using transformers tokenizer for chat templating
50
  llm_model = None # This will hold the llama_cpp.Llama instance
 
63
  print(f"Loading language model: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
64
  try:
65
  # --- Load Tokenizer (using transformers) ---
66
+ # Load the tokenizer from the original model repo
67
+ print(f"Loading tokenizer from original model repo: {ORIGINAL_MODEL_NAME}...")
68
+ tokenizer = AutoTokenizer.from_pretrained(ORIGINAL_MODEL_NAME)
 
 
 
69
 
70
  # Set pad_token if not already defined, often necessary for correct batching/generation behavior
71
  if tokenizer.pad_token is None:
 
81
 
82
  # --- Download GGUF model file (using huggingface_hub) ---
83
  print(f"Downloading GGUF model file: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
 
84
  model_path = hf_hub_download(
85
  repo_id=LLM_MODEL_REPO,
86
  filename=LLM_MODEL_FILE,
87
+ # cache_dir="/tmp/hf_cache" # Optional: specify a custom cache directory
88
  )
89
  print(f"GGUF model downloaded to: {model_path}")
90
 
 
92
  print(f"Loading GGUF model into llama_cpp...")
93
  # Instantiate the Llama model from the downloaded GGUF file
94
  # n_gpu_layers=0: Crucial for forcing CPU-only inference
95
+ # n_ctx: Context window size (tokens model can consider), match model's spec if possible (Qwen1.5 1.8B has 32768 context, but 4096 or 8192 is often sufficient and uses less RAM)
96
  # n_threads: Number of CPU threads to use. Set to your vCPU count (2) for better performance.
97
  llm_model = Llama(
98
  model_path=model_path,
99
  n_gpu_layers=0, # Explicitly use CPU
100
+ n_ctx=4096, # Context window size (adjust if needed)
101
+ n_threads=2 # Use 2 CPU threads
102
  )
103
  print("Llama.cpp model loaded successfully.")
104
 
 
113
  global caption_client
114
  print(f"Initializing Gradio client for {CAPTION_SPACE_URL}...")
115
  try:
116
+ # If the target Gradio Space requires authentication (e.g., private)
117
+ # store HF_TOKEN as a Space Secret and uncomment these lines.
 
118
  # HF_TOKEN = os.environ.get("HF_TOKEN")
119
  # if HF_TOKEN:
120
  # print("Using HF_TOKEN for Gradio client.")
 
123
  # print("HF_TOKEN not found. Initializing public Gradio client.")
124
  # caption_client = Client(CAPTION_SPACE_URL)
125
 
126
+ # Assuming the caption space is public
127
  caption_client = Client(CAPTION_SPACE_URL)
128
  print("Gradio client initialized successfully.")
129
  except Exception as e:
 
135
  # Load models and initialize clients when the app starts
136
  @app.on_event("startup")
137
  async def startup_event():
138
+ # Load the language model (Qwen1.5 1.8B GGUF via llama.cpp)
139
  load_language_model()
140
  # Initialize the client for the external captioning Space
141
  initialize_caption_client()
 
202
  print(f"Error removing temporary file {temp_file_path}: {e}") # Log cleanup errors
203
 
204
 
205
+ # --- Language Model Story Generation Function (Qwen1.5 1.8B via llama.cpp) ---
206
+ # Renamed function to reflect the model being used
207
+ def generate_story_qwen(prompt_text: str, max_new_tokens: int = 300, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 50) -> str:
208
  """
209
+ Generates text using the loaded Qwen1.5 1.8B model via llama.cpp.
210
  Uses the tokenizer to apply the chat template and calls llama.cpp's chat completion.
211
  """
212
  # Check if the language model was loaded successfully at startup
213
+ # Check for both tokenizer and llm_model (llama.cpp instance)
214
  if tokenizer is None or llm_model is None:
215
  # Raise a RuntimeError which is caught by the calling endpoint
216
  raise RuntimeError("Language model (llama.cpp) or tokenizer not loaded.")
217
 
218
+ # Construct the messages list following the chat format for Qwen1.5 Chat
219
+ # Qwen models use a standard ChatML-like format.
220
  messages = [
221
+ # System message is optional but can help guide the model's persona/style
222
+ # {"role": "system", "content": "You are a helpful and creative assistant."}
223
  {"role": "user", "content": prompt_text}
 
 
224
  ]
225
 
226
  try:
227
+ print("Calling llama.cpp create_chat_completion for Qwen...")
228
  # Call the create_chat_completion method from llama_cpp.Llama instance
229
+ # This method handles the chat templating internally for models like Qwen.
230
+ # max_tokens is the max number of tokens to generate
231
  # temperature, top_p control sampling. top_k might not be a direct parameter.
232
  response = llm_model.create_chat_completion(
233
  messages=messages,
 
236
  top_p=top_p,
237
  # top_k is sometimes supported, but check llama-cpp-python docs if needed
238
  # top_k=top_k,
239
+ stream=False # We want the full response at once
240
  )
241
+ print("Llama.cpp completion received for Qwen.")
242
 
243
  # Parse the response to get the generated text content
244
  # The response structure is typically like OpenAI's chat API response
 
246
  story = response['choices'][0].get('message', {}).get('content', '')
247
  else:
248
  # Handle cases where the response is empty or has an unexpected structure
249
+ print("Warning: Llama.cpp Qwen response structure unexpected or content missing.")
250
  story = "" # Return an empty string if content is not found
251
 
252
  except Exception as e:
253
  # Catch any exception that occurs during the llama.cpp inference process
254
+ print(f"Llama.cpp Qwen inference failed: {e}") # Log the error server-side
255
  # Re-raise as a RuntimeError to indicate failure to the endpoint
256
  raise RuntimeError(f"Llama.cpp inference failed: {type(e).__name__}: {e}")
257
 
 
285
  # This prompt instructs the model on what to write and incorporates the caption.
286
  prompt_text = f"Write an attractive story of around 300 words about {story_theme}. Incorporate the following details from an image description into the story: {caption}\n\nStory:"
287
 
288
+ # Step 3: Generate the story using the local language model (Qwen via llama.cpp)
289
  try:
290
+ # Call the Qwen story generation function
291
+ story = generate_story_qwen( # <--- Use the new function name
292
  prompt_text,
293
  max_new_tokens=300, # Request ~300 new tokens
294
  temperature=0.7, # Sampling parameters
 
298
  story = story.strip() # Basic cleanup of generated story text
299
 
300
  except RuntimeError as e:
301
+ # Catch specific RuntimeError raised by generate_story_qwen if LLM loading or inference fails
302
  print(f"Language model generation error: {e}") # Log the error server-side
303
  # Return a 503 Service Unavailable error if the LLM is not available or failed
304
  raise HTTPException(status_code=503, detail=f"Story generation failed (LLM): {e}")