import os import io import base64 from PIL import Image as PImage from transformers import pipeline import gradio as gr from dotenv import load_dotenv from openai import OpenAI import cv2 import numpy as np # ------------------------------- # Load environment variables # ------------------------------- load_dotenv() hf_api_key = os.getenv("HF_API_KEY") google_api_key = os.getenv("GOOGLE_API_KEY") if not hf_api_key or not google_api_key: raise ValueError("HF_API_KEY or GOOGLE_API_KEY not found in .env") os.environ["HF_API_KEY"] = hf_api_key # ------------------------------- # Initialize BLIP pipeline # ------------------------------- get_completion = pipeline("image-to-text", model="Salesforce/blip-image-captioning-large") # Detailed captions # ------------------------------- # Gemini client wrapper # ------------------------------- class GeminiClient: def __init__(self): self.gemini = OpenAI(api_key=google_api_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/") self.model_name = "gemini-2.0-flash" def chat(self, prompt_text): try: # Wrap prompt in proper message format messages = [{"role": "user", "content": prompt_text}] response = self.gemini.chat.completions.create(model=self.model_name, messages=messages) return response.choices[0].message.content except Exception as e: print(f"Gemini query failed: {e}") return "Sorry, I couldn't answer right now." gemini_client = GeminiClient() conversation_history = [] # ------------------------------- # Helper functions # ------------------------------- def image_to_base64_str(pil_image): byte_arr = io.BytesIO() pil_image.save(byte_arr, format='PNG') return base64.b64encode(byte_arr.getvalue()).decode('utf-8') def generate_caption(base64_image): image_data = base64.b64decode(base64_image) image = PImage.open(io.BytesIO(image_data)) result = get_completion(image) if isinstance(result, list) and len(result) > 0: return result[0].get('generated_text', "No caption generated") return "No caption generated" # ------------------------------- # Image modification using OpenCV # ------------------------------- def modify_image(image, style): img_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) if style.lower() == "sketch": gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY) inv = 255 - gray blur = cv2.GaussianBlur(inv, (21,21), 0) sketch = cv2.divide(gray, 255 - blur, scale=256) result = cv2.cvtColor(sketch, cv2.COLOR_GRAY2RGB) return PImage.fromarray(result) elif style.lower() == "gibli": gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY) gray = cv2.medianBlur(gray, 5) edges = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 9, 9) color = cv2.bilateralFilter(img_cv, 9, 300, 300) cartoon = cv2.bitwise_and(color, color, mask=edges) return PImage.fromarray(cv2.cvtColor(cartoon, cv2.COLOR_BGR2RGB)) else: return image # ------------------------------- # Gradio interface # ------------------------------- examples = ["christmas_dog.jpeg", "bird_flight.jpeg", "cow.jpeg", "dog_animal_greyhound.jpeg"] with gr.Blocks() as demo: gr.Markdown("# Multi-Feature Image AI") # Image Captioning with gr.Tab("Image Captioning"): image_input1 = gr.Image(label="Upload Image", type="pil", value=examples[0]) caption_output = gr.Textbox(label="Caption", lines=4, max_lines=12) caption_btn = gr.Button("Generate Caption") caption_btn.click(lambda img: generate_caption(image_to_base64_str(img)), inputs=image_input1, outputs=caption_output) # Chat with Image with gr.Tab("Chat with Image"): chatbot = gr.Chatbot(label="Chat with Image", type='messages') image_input2 = gr.Image(label="Upload Image", type="pil", value=examples[0]) user_input2 = gr.Textbox(label="Ask a question") state2 = gr.State() submit_btn2 = gr.Button("Send") def chat_fn(img, q, history): if history is None: history = [] caption_text = generate_caption(image_to_base64_str(img)) prompt_text = f"Detailed image description: {caption_text}. User question: {q}" answer = gemini_client.chat(prompt_text) # Append as proper dicts for Chatbot history.append({"role": "user", "content": q}) history.append({"role": "assistant", "content": answer}) messages = history[-10:] # last 10 messages return messages, history submit_btn2.click(chat_fn, inputs=[image_input2, user_input2, state2], outputs=[chatbot, state2]) # Modify Image with gr.Tab("Modify Image"): image_input3 = gr.Image(label="Upload Image", type="pil", value=examples[0]) style_input = gr.Dropdown(label="Style", choices=["Sketch", "Gibli"], value="Sketch") modified_output = gr.Image(label="Modified Image") modify_btn = gr.Button("Modify Image") modify_btn.click(modify_image, inputs=[image_input3, style_input], outputs=modified_output) demo.launch(share=True, server_port=int(os.environ.get('PORT', 7860)))