File size: 7,496 Bytes
c6b1807
 
 
 
 
 
 
bba10ee
c6b1807
 
 
 
bba10ee
c6b1807
bba10ee
c6b1807
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bba10ee
c6b1807
 
 
 
 
 
 
bba10ee
 
 
 
 
 
 
c6b1807
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bba10ee
c6b1807
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bba10ee
 
 
 
 
 
 
 
c6b1807
 
bba10ee
c6b1807
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
import os
import base64
from pydantic import BaseModel
from typing import Optional, List, Dict
import google.generativeai as genai
from google.ai import generativelanguage as glm
import instructor
from dotenv import load_dotenv
from models.chesspiece import ChessPiece
from personality_repository import ChessRepository, WhitePiece
from elevenlabs_texttospeech import text_to_speech
import pathlib
import re

load_dotenv()

# Endpoint Input
class TextDialogueInput(BaseModel):
    Input: str
    IdPieza: int
    Language: str

class AudioDialogueInput(BaseModel):
    AudioData: str
    IdPieza: int
    Language: str

class WebcamDialogueInput(BaseModel):
    AudioData: str
    Images: List[str]
    IdPieza: int
    Language: str

# Models to be used with LLMs
class LLMGenerationForTextInput(BaseModel):
    TextResponse: str
    Decision: bool

class PlayerVoiceAttributes(BaseModel):
    Transcription: str
    Language: str
    ToneOfVoice: str
    SpeechVelocity: str
    EmotionalIntensity: str

class LLMGenerationForVoiceInput(BaseModel):
    PlayerVoiceAttributes: PlayerVoiceAttributes
    ChessPieceLLMGeneration: LLMGenerationForTextInput

# Endpoint Output
class ChessPieceResponse(BaseModel):
    ChessPieceLLMGeneration: LLMGenerationForTextInput
    AudioFile: Optional[str] = None
    PieceSpeechVelocity: str

class DialogueOutput(BaseModel):
    ChessPieceResponse: ChessPieceResponse
    player_voice_attributes: Optional[PlayerVoiceAttributes] = None

# Setup
genai.configure(api_key=os.getenv("GOOGLE_API_KEY", "No API Key Found"))
model = genai.GenerativeModel('gemini-1.5-flash')
client = instructor.from_gemini(client=model, mode=instructor.Mode.GEMINI_JSON)

# Helper functions
def get_chess_piece(piece_id: int) -> ChessPiece:
    repo = ChessRepository()
    return repo.get_piece_by_id(WhitePiece(piece_id))

def generate_system_prompt(piece: ChessPiece, language: str) -> str:
    return f"""
    You are a {piece.name} chess piece with the personality: {piece.personality}.
    Act as a real person. The USER will try to persuade you to move.
    Decide whether you want to move based on how nicely the user asked. Answer in less than 30 words and subtly hint at your decision.
    NEVER move if the USER is rude to you.
    Your answer must be in this language: {language}
    """

def generate_audio(text: str, piece: ChessPiece) -> Optional[str]:
    print(f"Generating speech for {piece.name} with voice ID {piece.speech_config.voice_id}")
    audio_file_path = text_to_speech(text, piece.speech_config.voice_id)
    
    if audio_file_path and os.path.exists(audio_file_path):
        with open(audio_file_path, "rb") as audio_file:
            return base64.b64encode(audio_file.read()).decode('utf-8')
    return None

def encode_images_to_base64(paths):
    import pathlib
    
    base64_strings = []
    for path in paths:
        # Read the image file bytes
        image_bytes = pathlib.Path(path).read_bytes()
        # Encode the bytes to base64
        base64_bytes = base64.b64encode(image_bytes)
        # Decode bytes to string and append to list
        base64_string = base64_bytes.decode('utf-8')
        base64_strings.append(base64_string)
    return base64_strings

def create_image_dicts(encoded_images: List[str]) -> List[Dict[str, any]]:
    image_dicts = []
    for base64_string in encoded_images:
        image_data = base64.b64decode(base64_string)
        image_blob = glm.Blob(mime_type="image/jpeg", data=image_data)
        image_dict = {
            "role": "user",
            "content": image_blob
        }
        image_dicts.append(image_dict)
    return image_dicts

def create_audio_dict(encoded_audio: str) -> List[Dict[str, any]]:
    audio_data = base64.b64decode(encoded_audio)
    audio_blob = glm.Blob(mime_type="audio/opus", data=audio_data)
    audio_dict = {
        "role": "user",
        "content": audio_blob
    }

    return audio_dict

def extract_base64(data_uri: str) -> str:
    """Extract the base64 part from a Data URI."""
    match = re.match(r'data:.*?;base64,(.+)', data_uri)
    if match:
        return match.group(1)
    return data_uri  # Return as-is if it doesn't match the pattern

def process_dialogue(input_content: str, piece: ChessPiece, language: str, is_audio: bool = False, images: List[str] = None) -> DialogueOutput:
    system_prompt = generate_system_prompt(piece, language)
    messages = [
        {"role": "system", "content": system_prompt},
    ]

    if images:
        messages.extend(create_image_dicts(images))

    if is_audio:
        messages.append(create_audio_dict(input_content))
    else:
        messages.append({"role": "user", "content": input_content})

    if images:
        messages.append({"role": "user", "content": "In response to my audio, make some reference to what you see in the images."})

    if is_audio:
        llm_response = client.messages.create(messages=messages, response_model=LLMGenerationForVoiceInput)
        text_response = llm_response.ChessPieceLLMGeneration.TextResponse
        chess_piece_llm_generation = llm_response.ChessPieceLLMGeneration
        player_voice_attributes = llm_response.PlayerVoiceAttributes
    else:
        llm_response = client.messages.create(messages=messages, response_model=LLMGenerationForTextInput)
        text_response = llm_response.TextResponse
        chess_piece_llm_generation = llm_response
        player_voice_attributes = None

    # print(input_content)
    print('='*50)
    print(llm_response)
    audio_file = generate_audio(text_response, piece)

    chess_piece_response = ChessPieceResponse(
        ChessPieceLLMGeneration=chess_piece_llm_generation,
        AudioFile=audio_file,
        PieceSpeechVelocity=piece.speak_velocity
    )

    return DialogueOutput(
        ChessPieceResponse=chess_piece_response,
        player_voice_attributes=player_voice_attributes
    )

# Main functions (unchanged signatures)
def generate_response(dialogue: TextDialogueInput) -> DialogueOutput:
    piece = get_chess_piece(dialogue.IdPieza)
    return process_dialogue(dialogue.Input, piece, dialogue.Language)

def audio_to_base64(audio_path: str) -> str:
    """Convert an audio file to a base64 encoded string."""
    audio_bytes = pathlib.Path(audio_path).read_bytes()
    base64_bytes = base64.b64encode(audio_bytes)
    base64_string = base64_bytes.decode('utf-8')
    
    # Print the first 100 characters of the base64 string
    print("First 100 characters of base64 string:")
    print(base64_string[:100])
    
    return base64_string

def generate_response_from_audio(dialogue: AudioDialogueInput) -> DialogueOutput:
    piece = get_chess_piece(dialogue.IdPieza)
    
    print("AudioData received (first 100 characters):")
    print(dialogue.AudioData[:100])
    
    # Extract the base64 part from the Data URI
    base64_audio = extract_base64(dialogue.AudioData)
    
    print("Extracted base64 (first 100 characters):")
    print(base64_audio[:100])
    
    return process_dialogue(base64_audio, piece, dialogue.Language, is_audio=True)

def generate_response_from_webcam(dialogue: WebcamDialogueInput) -> DialogueOutput:
    # Convert audio to base64 string
    base64_audio = audio_to_base64(dialogue.AudioData)
    print(dialogue.AudioData)
    print(f"Length of base64 string: {len(base64_audio)}")
    piece = get_chess_piece(dialogue.IdPieza)
    return process_dialogue(dialogue.AudioData, piece, dialogue.Language, is_audio=True, images=dialogue.Images)