File size: 5,473 Bytes
29efbc6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | import os
import io
import base64
from PIL import Image as PImage
from transformers import pipeline
import gradio as gr
from dotenv import load_dotenv
from openai import OpenAI
import cv2
import numpy as np
# -------------------------------
# Load environment variables
# -------------------------------
load_dotenv()
hf_api_key = os.getenv("HF_API_KEY")
google_api_key = os.getenv("GOOGLE_API_KEY")
if not hf_api_key or not google_api_key:
raise ValueError("HF_API_KEY or GOOGLE_API_KEY not found in .env")
os.environ["HF_API_KEY"] = hf_api_key
# -------------------------------
# Initialize BLIP pipeline
# -------------------------------
get_completion = pipeline("image-to-text", model="Salesforce/blip-image-captioning-large") # Detailed captions
# -------------------------------
# Gemini client wrapper
# -------------------------------
class GeminiClient:
def __init__(self):
self.gemini = OpenAI(api_key=google_api_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/")
self.model_name = "gemini-2.0-flash"
def chat(self, prompt_text):
try:
# Wrap prompt in proper message format
messages = [{"role": "user", "content": prompt_text}]
response = self.gemini.chat.completions.create(model=self.model_name, messages=messages)
return response.choices[0].message.content
except Exception as e:
print(f"Gemini query failed: {e}")
return "Sorry, I couldn't answer right now."
gemini_client = GeminiClient()
conversation_history = []
# -------------------------------
# Helper functions
# -------------------------------
def image_to_base64_str(pil_image):
byte_arr = io.BytesIO()
pil_image.save(byte_arr, format='PNG')
return base64.b64encode(byte_arr.getvalue()).decode('utf-8')
def generate_caption(base64_image):
image_data = base64.b64decode(base64_image)
image = PImage.open(io.BytesIO(image_data))
result = get_completion(image)
if isinstance(result, list) and len(result) > 0:
return result[0].get('generated_text', "No caption generated")
return "No caption generated"
# -------------------------------
# Image modification using OpenCV
# -------------------------------
def modify_image(image, style):
img_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
if style.lower() == "sketch":
gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY)
inv = 255 - gray
blur = cv2.GaussianBlur(inv, (21,21), 0)
sketch = cv2.divide(gray, 255 - blur, scale=256)
result = cv2.cvtColor(sketch, cv2.COLOR_GRAY2RGB)
return PImage.fromarray(result)
elif style.lower() == "gibli":
gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY)
gray = cv2.medianBlur(gray, 5)
edges = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 9, 9)
color = cv2.bilateralFilter(img_cv, 9, 300, 300)
cartoon = cv2.bitwise_and(color, color, mask=edges)
return PImage.fromarray(cv2.cvtColor(cartoon, cv2.COLOR_BGR2RGB))
else:
return image
# -------------------------------
# Gradio interface
# -------------------------------
examples = ["christmas_dog.jpeg", "bird_flight.jpeg", "cow.jpeg", "dog_animal_greyhound.jpeg"]
with gr.Blocks() as demo:
gr.Markdown("# Multi-Feature Image AI")
# Image Captioning
with gr.Tab("Image Captioning"):
image_input1 = gr.Image(label="Upload Image", type="pil", value=examples[0])
caption_output = gr.Textbox(label="Caption", lines=4, max_lines=12)
caption_btn = gr.Button("Generate Caption")
caption_btn.click(lambda img: generate_caption(image_to_base64_str(img)), inputs=image_input1, outputs=caption_output)
# Chat with Image
with gr.Tab("Chat with Image"):
chatbot = gr.Chatbot(label="Chat with Image", type='messages')
image_input2 = gr.Image(label="Upload Image", type="pil", value=examples[0])
user_input2 = gr.Textbox(label="Ask a question")
state2 = gr.State()
submit_btn2 = gr.Button("Send")
def chat_fn(img, q, history):
if history is None:
history = []
caption_text = generate_caption(image_to_base64_str(img))
prompt_text = f"Detailed image description: {caption_text}. User question: {q}"
answer = gemini_client.chat(prompt_text)
# Append as proper dicts for Chatbot
history.append({"role": "user", "content": q})
history.append({"role": "assistant", "content": answer})
messages = history[-10:] # last 10 messages
return messages, history
submit_btn2.click(chat_fn, inputs=[image_input2, user_input2, state2], outputs=[chatbot, state2])
# Modify Image
with gr.Tab("Modify Image"):
image_input3 = gr.Image(label="Upload Image", type="pil", value=examples[0])
style_input = gr.Dropdown(label="Style", choices=["Sketch", "Gibli"], value="Sketch")
modified_output = gr.Image(label="Modified Image")
modify_btn = gr.Button("Modify Image")
modify_btn.click(modify_image, inputs=[image_input3, style_input], outputs=modified_output)
demo.launch(share=True, server_port=int(os.environ.get('PORT', 7860))) |