Update main.py
Browse files
main.py
CHANGED
|
@@ -5,14 +5,15 @@ from fastapi import FastAPI, File, UploadFile, Form, HTTPException
|
|
| 5 |
# from fastapi.templating import Jinja2Templates
|
| 6 |
# from fastapi.responses import FileResponse
|
| 7 |
|
| 8 |
-
# Removed 'requests'
|
| 9 |
# import requests
|
| 10 |
import base64 # Keep if needed elsewhere (not strictly needed for this version)
|
| 11 |
import os
|
| 12 |
import random
|
| 13 |
-
|
|
|
|
| 14 |
|
| 15 |
-
# Import necessary classes from transformers (Keeping only AutoTokenizer
|
| 16 |
from transformers import AutoTokenizer
|
| 17 |
|
| 18 |
# Import necessary modules for llama-cpp-python and downloading from Hub
|
|
@@ -36,10 +37,14 @@ from deep_translator.exceptions import InvalidSourceOrTargetLanguage
|
|
| 36 |
app = FastAPI()
|
| 37 |
|
| 38 |
# --- Llama.cpp Language Model Setup (Local CPU Inference) ---
|
| 39 |
-
# Repository on Hugging Face Hub containing the GGUF file
|
| 40 |
-
#
|
| 41 |
-
LLM_MODEL_REPO = "TheBloke/
|
| 42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
tokenizer = None # Using transformers tokenizer for chat templating
|
| 45 |
llm_model = None # This will hold the llama_cpp.Llama instance
|
|
@@ -58,12 +63,9 @@ def load_language_model():
|
|
| 58 |
print(f"Loading language model: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
|
| 59 |
try:
|
| 60 |
# --- Load Tokenizer (using transformers) ---
|
| 61 |
-
#
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
original_model_name = LLM_MODEL_REPO.split('/')[1].replace('-GGUF', '')
|
| 65 |
-
print(f"Loading tokenizer from original model repo: TinyLlama/{original_model_name}...")
|
| 66 |
-
tokenizer = AutoTokenizer.from_pretrained(f"TinyLlama/{original_model_name}")
|
| 67 |
|
| 68 |
# Set pad_token if not already defined, often necessary for correct batching/generation behavior
|
| 69 |
if tokenizer.pad_token is None:
|
|
@@ -79,11 +81,10 @@ def load_language_model():
|
|
| 79 |
|
| 80 |
# --- Download GGUF model file (using huggingface_hub) ---
|
| 81 |
print(f"Downloading GGUF model file: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
|
| 82 |
-
# hf_hub_download downloads the file to the Hugging Face cache directory
|
| 83 |
model_path = hf_hub_download(
|
| 84 |
repo_id=LLM_MODEL_REPO,
|
| 85 |
filename=LLM_MODEL_FILE,
|
| 86 |
-
# cache_dir="/tmp/hf_cache" # Optional: specify a custom cache directory
|
| 87 |
)
|
| 88 |
print(f"GGUF model downloaded to: {model_path}")
|
| 89 |
|
|
@@ -91,13 +92,13 @@ def load_language_model():
|
|
| 91 |
print(f"Loading GGUF model into llama_cpp...")
|
| 92 |
# Instantiate the Llama model from the downloaded GGUF file
|
| 93 |
# n_gpu_layers=0: Crucial for forcing CPU-only inference
|
| 94 |
-
# n_ctx: Context window size (tokens model can consider), match model's spec if possible
|
| 95 |
# n_threads: Number of CPU threads to use. Set to your vCPU count (2) for better performance.
|
| 96 |
llm_model = Llama(
|
| 97 |
model_path=model_path,
|
| 98 |
n_gpu_layers=0, # Explicitly use CPU
|
| 99 |
-
n_ctx=4096, # Context window size (
|
| 100 |
-
n_threads=2 # Use 2 CPU threads
|
| 101 |
)
|
| 102 |
print("Llama.cpp model loaded successfully.")
|
| 103 |
|
|
@@ -112,9 +113,8 @@ def initialize_caption_client():
|
|
| 112 |
global caption_client
|
| 113 |
print(f"Initializing Gradio client for {CAPTION_SPACE_URL}...")
|
| 114 |
try:
|
| 115 |
-
# If the target Gradio Space requires authentication (e.g.,
|
| 116 |
-
#
|
| 117 |
-
# Store this token as a Space Secret named HF_TOKEN.
|
| 118 |
# HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 119 |
# if HF_TOKEN:
|
| 120 |
# print("Using HF_TOKEN for Gradio client.")
|
|
@@ -123,7 +123,7 @@ def initialize_caption_client():
|
|
| 123 |
# print("HF_TOKEN not found. Initializing public Gradio client.")
|
| 124 |
# caption_client = Client(CAPTION_SPACE_URL)
|
| 125 |
|
| 126 |
-
# Assuming the caption space is public
|
| 127 |
caption_client = Client(CAPTION_SPACE_URL)
|
| 128 |
print("Gradio client initialized successfully.")
|
| 129 |
except Exception as e:
|
|
@@ -135,7 +135,7 @@ def initialize_caption_client():
|
|
| 135 |
# Load models and initialize clients when the app starts
|
| 136 |
@app.on_event("startup")
|
| 137 |
async def startup_event():
|
| 138 |
-
# Load the language model (
|
| 139 |
load_language_model()
|
| 140 |
# Initialize the client for the external captioning Space
|
| 141 |
initialize_caption_client()
|
|
@@ -202,29 +202,32 @@ def generate_image_caption(image_file: UploadFile):
|
|
| 202 |
print(f"Error removing temporary file {temp_file_path}: {e}") # Log cleanup errors
|
| 203 |
|
| 204 |
|
| 205 |
-
# --- Language Model Story Generation Function (
|
| 206 |
-
|
|
|
|
| 207 |
"""
|
| 208 |
-
Generates text using the loaded
|
| 209 |
Uses the tokenizer to apply the chat template and calls llama.cpp's chat completion.
|
| 210 |
"""
|
| 211 |
# Check if the language model was loaded successfully at startup
|
|
|
|
| 212 |
if tokenizer is None or llm_model is None:
|
| 213 |
# Raise a RuntimeError which is caught by the calling endpoint
|
| 214 |
raise RuntimeError("Language model (llama.cpp) or tokenizer not loaded.")
|
| 215 |
|
| 216 |
-
# Construct the messages list following the chat format
|
|
|
|
| 217 |
messages = [
|
|
|
|
|
|
|
| 218 |
{"role": "user", "content": prompt_text}
|
| 219 |
-
# You could add a system message here if the model was trained with one
|
| 220 |
-
# e.g., {"role": "system", "content": "You are a creative storyteller."}
|
| 221 |
]
|
| 222 |
|
| 223 |
try:
|
| 224 |
-
print("Calling llama.cpp create_chat_completion...")
|
| 225 |
# Call the create_chat_completion method from llama_cpp.Llama instance
|
| 226 |
-
# This method handles the chat templating internally for
|
| 227 |
-
# max_tokens is the max number of tokens to generate
|
| 228 |
# temperature, top_p control sampling. top_k might not be a direct parameter.
|
| 229 |
response = llm_model.create_chat_completion(
|
| 230 |
messages=messages,
|
|
@@ -233,9 +236,9 @@ def generate_story_tinyllama(prompt_text: str, max_new_tokens: int = 300, temper
|
|
| 233 |
top_p=top_p,
|
| 234 |
# top_k is sometimes supported, but check llama-cpp-python docs if needed
|
| 235 |
# top_k=top_k,
|
| 236 |
-
stream=False # We want the full response at once
|
| 237 |
)
|
| 238 |
-
print("Llama.cpp completion received.")
|
| 239 |
|
| 240 |
# Parse the response to get the generated text content
|
| 241 |
# The response structure is typically like OpenAI's chat API response
|
|
@@ -243,12 +246,12 @@ def generate_story_tinyllama(prompt_text: str, max_new_tokens: int = 300, temper
|
|
| 243 |
story = response['choices'][0].get('message', {}).get('content', '')
|
| 244 |
else:
|
| 245 |
# Handle cases where the response is empty or has an unexpected structure
|
| 246 |
-
print("Warning: Llama.cpp response structure unexpected or content missing.")
|
| 247 |
story = "" # Return an empty string if content is not found
|
| 248 |
|
| 249 |
except Exception as e:
|
| 250 |
# Catch any exception that occurs during the llama.cpp inference process
|
| 251 |
-
print(f"Llama.cpp inference failed: {e}") # Log the error server-side
|
| 252 |
# Re-raise as a RuntimeError to indicate failure to the endpoint
|
| 253 |
raise RuntimeError(f"Llama.cpp inference failed: {type(e).__name__}: {e}")
|
| 254 |
|
|
@@ -282,10 +285,10 @@ async def generate_story_endpoint(image_file: UploadFile = File(...), language:
|
|
| 282 |
# This prompt instructs the model on what to write and incorporates the caption.
|
| 283 |
prompt_text = f"Write an attractive story of around 300 words about {story_theme}. Incorporate the following details from an image description into the story: {caption}\n\nStory:"
|
| 284 |
|
| 285 |
-
# Step 3: Generate the story using the local language model (
|
| 286 |
try:
|
| 287 |
-
# Call the story generation function
|
| 288 |
-
story =
|
| 289 |
prompt_text,
|
| 290 |
max_new_tokens=300, # Request ~300 new tokens
|
| 291 |
temperature=0.7, # Sampling parameters
|
|
@@ -295,7 +298,7 @@ async def generate_story_endpoint(image_file: UploadFile = File(...), language:
|
|
| 295 |
story = story.strip() # Basic cleanup of generated story text
|
| 296 |
|
| 297 |
except RuntimeError as e:
|
| 298 |
-
# Catch specific RuntimeError raised by
|
| 299 |
print(f"Language model generation error: {e}") # Log the error server-side
|
| 300 |
# Return a 503 Service Unavailable error if the LLM is not available or failed
|
| 301 |
raise HTTPException(status_code=503, detail=f"Story generation failed (LLM): {e}")
|
|
|
|
| 5 |
# from fastapi.templating import Jinja2Templates
|
| 6 |
# from fastapi.responses import FileResponse
|
| 7 |
|
| 8 |
+
# Removed 'requests'
|
| 9 |
# import requests
|
| 10 |
import base64 # Keep if needed elsewhere (not strictly needed for this version)
|
| 11 |
import os
|
| 12 |
import random
|
| 13 |
+
# Removed unused IO import
|
| 14 |
+
# from typing import IO
|
| 15 |
|
| 16 |
+
# Import necessary classes from transformers (Keeping only AutoTokenizer)
|
| 17 |
from transformers import AutoTokenizer
|
| 18 |
|
| 19 |
# Import necessary modules for llama-cpp-python and downloading from Hub
|
|
|
|
| 37 |
app = FastAPI()
|
| 38 |
|
| 39 |
# --- Llama.cpp Language Model Setup (Local CPU Inference) ---
|
| 40 |
+
# Repository on Hugging Face Hub containing the Qwen1.5 1.8B GGUF file
|
| 41 |
+
# Using TheBloke's repo as a common source for GGUF models.
|
| 42 |
+
LLM_MODEL_REPO = "TheBloke/Qwen1.5-1.8B-Chat-GGUF"
|
| 43 |
+
# Specify the filename for a Q4_K_M quantized version (good balance of speed/quality on CPU)
|
| 44 |
+
LLM_MODEL_FILE = "qwen1_5-1_8b-chat-Q4_K_M.gguf"
|
| 45 |
+
|
| 46 |
+
# Original model name for the tokenizer (needed by transformers)
|
| 47 |
+
ORIGINAL_MODEL_NAME = "Qwen/Qwen1.5-1.8B-Chat"
|
| 48 |
|
| 49 |
tokenizer = None # Using transformers tokenizer for chat templating
|
| 50 |
llm_model = None # This will hold the llama_cpp.Llama instance
|
|
|
|
| 63 |
print(f"Loading language model: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
|
| 64 |
try:
|
| 65 |
# --- Load Tokenizer (using transformers) ---
|
| 66 |
+
# Load the tokenizer from the original model repo
|
| 67 |
+
print(f"Loading tokenizer from original model repo: {ORIGINAL_MODEL_NAME}...")
|
| 68 |
+
tokenizer = AutoTokenizer.from_pretrained(ORIGINAL_MODEL_NAME)
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
# Set pad_token if not already defined, often necessary for correct batching/generation behavior
|
| 71 |
if tokenizer.pad_token is None:
|
|
|
|
| 81 |
|
| 82 |
# --- Download GGUF model file (using huggingface_hub) ---
|
| 83 |
print(f"Downloading GGUF model file: {LLM_MODEL_FILE} from {LLM_MODEL_REPO}...")
|
|
|
|
| 84 |
model_path = hf_hub_download(
|
| 85 |
repo_id=LLM_MODEL_REPO,
|
| 86 |
filename=LLM_MODEL_FILE,
|
| 87 |
+
# cache_dir="/tmp/hf_cache" # Optional: specify a custom cache directory
|
| 88 |
)
|
| 89 |
print(f"GGUF model downloaded to: {model_path}")
|
| 90 |
|
|
|
|
| 92 |
print(f"Loading GGUF model into llama_cpp...")
|
| 93 |
# Instantiate the Llama model from the downloaded GGUF file
|
| 94 |
# n_gpu_layers=0: Crucial for forcing CPU-only inference
|
| 95 |
+
# n_ctx: Context window size (tokens model can consider), match model's spec if possible (Qwen1.5 1.8B has 32768 context, but 4096 or 8192 is often sufficient and uses less RAM)
|
| 96 |
# n_threads: Number of CPU threads to use. Set to your vCPU count (2) for better performance.
|
| 97 |
llm_model = Llama(
|
| 98 |
model_path=model_path,
|
| 99 |
n_gpu_layers=0, # Explicitly use CPU
|
| 100 |
+
n_ctx=4096, # Context window size (adjust if needed)
|
| 101 |
+
n_threads=2 # Use 2 CPU threads
|
| 102 |
)
|
| 103 |
print("Llama.cpp model loaded successfully.")
|
| 104 |
|
|
|
|
| 113 |
global caption_client
|
| 114 |
print(f"Initializing Gradio client for {CAPTION_SPACE_URL}...")
|
| 115 |
try:
|
| 116 |
+
# If the target Gradio Space requires authentication (e.g., private)
|
| 117 |
+
# store HF_TOKEN as a Space Secret and uncomment these lines.
|
|
|
|
| 118 |
# HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 119 |
# if HF_TOKEN:
|
| 120 |
# print("Using HF_TOKEN for Gradio client.")
|
|
|
|
| 123 |
# print("HF_TOKEN not found. Initializing public Gradio client.")
|
| 124 |
# caption_client = Client(CAPTION_SPACE_URL)
|
| 125 |
|
| 126 |
+
# Assuming the caption space is public
|
| 127 |
caption_client = Client(CAPTION_SPACE_URL)
|
| 128 |
print("Gradio client initialized successfully.")
|
| 129 |
except Exception as e:
|
|
|
|
| 135 |
# Load models and initialize clients when the app starts
|
| 136 |
@app.on_event("startup")
|
| 137 |
async def startup_event():
|
| 138 |
+
# Load the language model (Qwen1.5 1.8B GGUF via llama.cpp)
|
| 139 |
load_language_model()
|
| 140 |
# Initialize the client for the external captioning Space
|
| 141 |
initialize_caption_client()
|
|
|
|
| 202 |
print(f"Error removing temporary file {temp_file_path}: {e}") # Log cleanup errors
|
| 203 |
|
| 204 |
|
| 205 |
+
# --- Language Model Story Generation Function (Qwen1.5 1.8B via llama.cpp) ---
|
| 206 |
+
# Renamed function to reflect the model being used
|
| 207 |
+
def generate_story_qwen(prompt_text: str, max_new_tokens: int = 300, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 50) -> str:
|
| 208 |
"""
|
| 209 |
+
Generates text using the loaded Qwen1.5 1.8B model via llama.cpp.
|
| 210 |
Uses the tokenizer to apply the chat template and calls llama.cpp's chat completion.
|
| 211 |
"""
|
| 212 |
# Check if the language model was loaded successfully at startup
|
| 213 |
+
# Check for both tokenizer and llm_model (llama.cpp instance)
|
| 214 |
if tokenizer is None or llm_model is None:
|
| 215 |
# Raise a RuntimeError which is caught by the calling endpoint
|
| 216 |
raise RuntimeError("Language model (llama.cpp) or tokenizer not loaded.")
|
| 217 |
|
| 218 |
+
# Construct the messages list following the chat format for Qwen1.5 Chat
|
| 219 |
+
# Qwen models use a standard ChatML-like format.
|
| 220 |
messages = [
|
| 221 |
+
# System message is optional but can help guide the model's persona/style
|
| 222 |
+
# {"role": "system", "content": "You are a helpful and creative assistant."}
|
| 223 |
{"role": "user", "content": prompt_text}
|
|
|
|
|
|
|
| 224 |
]
|
| 225 |
|
| 226 |
try:
|
| 227 |
+
print("Calling llama.cpp create_chat_completion for Qwen...")
|
| 228 |
# Call the create_chat_completion method from llama_cpp.Llama instance
|
| 229 |
+
# This method handles the chat templating internally for models like Qwen.
|
| 230 |
+
# max_tokens is the max number of tokens to generate
|
| 231 |
# temperature, top_p control sampling. top_k might not be a direct parameter.
|
| 232 |
response = llm_model.create_chat_completion(
|
| 233 |
messages=messages,
|
|
|
|
| 236 |
top_p=top_p,
|
| 237 |
# top_k is sometimes supported, but check llama-cpp-python docs if needed
|
| 238 |
# top_k=top_k,
|
| 239 |
+
stream=False # We want the full response at once
|
| 240 |
)
|
| 241 |
+
print("Llama.cpp completion received for Qwen.")
|
| 242 |
|
| 243 |
# Parse the response to get the generated text content
|
| 244 |
# The response structure is typically like OpenAI's chat API response
|
|
|
|
| 246 |
story = response['choices'][0].get('message', {}).get('content', '')
|
| 247 |
else:
|
| 248 |
# Handle cases where the response is empty or has an unexpected structure
|
| 249 |
+
print("Warning: Llama.cpp Qwen response structure unexpected or content missing.")
|
| 250 |
story = "" # Return an empty string if content is not found
|
| 251 |
|
| 252 |
except Exception as e:
|
| 253 |
# Catch any exception that occurs during the llama.cpp inference process
|
| 254 |
+
print(f"Llama.cpp Qwen inference failed: {e}") # Log the error server-side
|
| 255 |
# Re-raise as a RuntimeError to indicate failure to the endpoint
|
| 256 |
raise RuntimeError(f"Llama.cpp inference failed: {type(e).__name__}: {e}")
|
| 257 |
|
|
|
|
| 285 |
# This prompt instructs the model on what to write and incorporates the caption.
|
| 286 |
prompt_text = f"Write an attractive story of around 300 words about {story_theme}. Incorporate the following details from an image description into the story: {caption}\n\nStory:"
|
| 287 |
|
| 288 |
+
# Step 3: Generate the story using the local language model (Qwen via llama.cpp)
|
| 289 |
try:
|
| 290 |
+
# Call the Qwen story generation function
|
| 291 |
+
story = generate_story_qwen( # <--- Use the new function name
|
| 292 |
prompt_text,
|
| 293 |
max_new_tokens=300, # Request ~300 new tokens
|
| 294 |
temperature=0.7, # Sampling parameters
|
|
|
|
| 298 |
story = story.strip() # Basic cleanup of generated story text
|
| 299 |
|
| 300 |
except RuntimeError as e:
|
| 301 |
+
# Catch specific RuntimeError raised by generate_story_qwen if LLM loading or inference fails
|
| 302 |
print(f"Language model generation error: {e}") # Log the error server-side
|
| 303 |
# Return a 503 Service Unavailable error if the LLM is not available or failed
|
| 304 |
raise HTTPException(status_code=503, detail=f"Story generation failed (LLM): {e}")
|