File size: 5,473 Bytes
29efbc6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import os
import io
import base64
from PIL import Image as PImage
from transformers import pipeline
import gradio as gr
from dotenv import load_dotenv
from openai import OpenAI
import cv2
import numpy as np

# -------------------------------
# Load environment variables
# -------------------------------
load_dotenv()
hf_api_key = os.getenv("HF_API_KEY")
google_api_key = os.getenv("GOOGLE_API_KEY")

if not hf_api_key or not google_api_key:
    raise ValueError("HF_API_KEY or GOOGLE_API_KEY not found in .env")

os.environ["HF_API_KEY"] = hf_api_key

# -------------------------------
# Initialize BLIP pipeline
# -------------------------------
get_completion = pipeline("image-to-text", model="Salesforce/blip-image-captioning-large")  # Detailed captions

# -------------------------------
# Gemini client wrapper
# -------------------------------
class GeminiClient:
    def __init__(self):
        self.gemini = OpenAI(api_key=google_api_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/")
        self.model_name = "gemini-2.0-flash"

    def chat(self, prompt_text):
        try:
            # Wrap prompt in proper message format
            messages = [{"role": "user", "content": prompt_text}]
            response = self.gemini.chat.completions.create(model=self.model_name, messages=messages)
            return response.choices[0].message.content
        except Exception as e:
            print(f"Gemini query failed: {e}")
            return "Sorry, I couldn't answer right now."

gemini_client = GeminiClient()
conversation_history = []

# -------------------------------
# Helper functions
# -------------------------------
def image_to_base64_str(pil_image):
    byte_arr = io.BytesIO()
    pil_image.save(byte_arr, format='PNG')
    return base64.b64encode(byte_arr.getvalue()).decode('utf-8')

def generate_caption(base64_image):
    image_data = base64.b64decode(base64_image)
    image = PImage.open(io.BytesIO(image_data))
    result = get_completion(image)
    if isinstance(result, list) and len(result) > 0:
        return result[0].get('generated_text', "No caption generated")
    return "No caption generated"

# -------------------------------
# Image modification using OpenCV
# -------------------------------
def modify_image(image, style):
    img_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
    if style.lower() == "sketch":
        gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY)
        inv = 255 - gray
        blur = cv2.GaussianBlur(inv, (21,21), 0)
        sketch = cv2.divide(gray, 255 - blur, scale=256)
        result = cv2.cvtColor(sketch, cv2.COLOR_GRAY2RGB)
        return PImage.fromarray(result)
    elif style.lower() == "gibli":
        gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY)
        gray = cv2.medianBlur(gray, 5)
        edges = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 9, 9)
        color = cv2.bilateralFilter(img_cv, 9, 300, 300)
        cartoon = cv2.bitwise_and(color, color, mask=edges)
        return PImage.fromarray(cv2.cvtColor(cartoon, cv2.COLOR_BGR2RGB))
    else:
        return image

# -------------------------------
# Gradio interface
# -------------------------------
examples = ["christmas_dog.jpeg", "bird_flight.jpeg", "cow.jpeg", "dog_animal_greyhound.jpeg"]

with gr.Blocks() as demo:
    gr.Markdown("# Multi-Feature Image AI")

    # Image Captioning
    with gr.Tab("Image Captioning"):
        image_input1 = gr.Image(label="Upload Image", type="pil", value=examples[0])
        caption_output = gr.Textbox(label="Caption", lines=4, max_lines=12)
        caption_btn = gr.Button("Generate Caption")
        caption_btn.click(lambda img: generate_caption(image_to_base64_str(img)), inputs=image_input1, outputs=caption_output)

    # Chat with Image
    with gr.Tab("Chat with Image"):
        chatbot = gr.Chatbot(label="Chat with Image", type='messages')
        image_input2 = gr.Image(label="Upload Image", type="pil", value=examples[0])
        user_input2 = gr.Textbox(label="Ask a question")
        state2 = gr.State()
        submit_btn2 = gr.Button("Send")
        
        def chat_fn(img, q, history):
            if history is None:
                history = []
            caption_text = generate_caption(image_to_base64_str(img))
            prompt_text = f"Detailed image description: {caption_text}. User question: {q}"
            answer = gemini_client.chat(prompt_text)
            # Append as proper dicts for Chatbot
            history.append({"role": "user", "content": q})
            history.append({"role": "assistant", "content": answer})
            messages = history[-10:]  # last 10 messages
            return messages, history

        submit_btn2.click(chat_fn, inputs=[image_input2, user_input2, state2], outputs=[chatbot, state2])

    # Modify Image
    with gr.Tab("Modify Image"):
        image_input3 = gr.Image(label="Upload Image", type="pil", value=examples[0])
        style_input = gr.Dropdown(label="Style", choices=["Sketch", "Gibli"], value="Sketch")
        modified_output = gr.Image(label="Modified Image")
        modify_btn = gr.Button("Modify Image")
        modify_btn.click(modify_image, inputs=[image_input3, style_input], outputs=modified_output)


demo.launch(share=True, server_port=int(os.environ.get('PORT', 7860)))