Spaces:
Sleeping
Sleeping
| import openai | |
| import base64 | |
| import cv2 | |
| import gradio as gr | |
| # import os | |
| # api_key = os.getenv("OPENAI_API_KEY") | |
| # openai.api_key = api_key | |
| # Function to encode a numpy array image | |
| def encode_image(image_array): | |
| _, buffer = cv2.imencode('.jpg', image_array) | |
| return base64.b64encode(buffer).decode("utf-8") | |
| def respond_with_audio(image_array, text_prompt, voice='alloy'): | |
| # Encode the image | |
| base64_image = encode_image(image_array) | |
| # Step 1: Get response from GPT-4o | |
| response = openai.chat.completions.create( | |
| model="gpt-4o", | |
| messages=[ | |
| {"role": "system", "content": "Describe and respond to the user's input."}, | |
| {"role": "user", "content": [ | |
| {"type": "text", "text": text_prompt}, | |
| {"type": "image_url", "image_url": { | |
| "url": f"data:image/jpeg;base64,{base64_image}", | |
| }}, | |
| ]} | |
| ], | |
| max_tokens=300 | |
| ) | |
| message_text = response.choices[0].message.content | |
| # Step 2: Convert text to speech | |
| tts_response = openai.audio.speech.create( | |
| model="tts-1", | |
| voice=voice, # e.g., 'alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer' | |
| input=message_text | |
| ) | |
| tts_response_ = tts_response.content | |
| return message_text, tts_response_ | |
| with gr.Blocks() as block: | |
| with gr.Row(): | |
| gr.HTML( | |
| f""" | |
| <h1 style='text-align: center;'> Kid's Bot 👼</h1> | |
| <h3 style='text-align: center;'> Hint: change the question </h3> | |
| <p style='text-align: center;'> Created by <a href="https://github.com/pacomesimon"> Pacome</a> | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| image_input = gr.Image(label="Capture Image", sources=["webcam"], type="numpy") | |
| examples = gr.Examples( | |
| examples=["small_image.jpg"], | |
| inputs=[image_input], | |
| ) | |
| text_prompt = gr.Textbox(label="Enter your question", | |
| value="what is this image showing? explain it to a 5 years old kid", lines=1) | |
| with gr.Column(): | |
| # audio_out = gr.Audio(label="Spoken Answer", streaming=True, autoplay=True, loop=False) | |
| audio_out = gr.Audio(label="Spoken Answer", autoplay=True) | |
| answer = gr.Textbox(label="Answer") | |
| def process_input(image, text): | |
| # print(f"Processing input Q: {text}") | |
| message_text, tts_content = respond_with_audio(image, text) | |
| # print(f"Answer: {message_text}") | |
| # print(f"Audio: {[tts_content]}") | |
| return message_text, tts_content | |
| image_input.change(fn=process_input, inputs=[image_input, text_prompt], outputs=[answer, audio_out]) | |
| block.launch( | |
| debug=True, | |
| share=True, | |
| ) | |