AutoSDR / LLMClient.py
shkkonda's picture
Upload 6 files
3319147 verified
Raw History Blame Contribute Delete
15.1 kB
import openai
import json
from typing import Dict, List, Optional, Any, Union
import asyncio
from langchain_core.language_models.llms import LLM
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.callbacks.manager import CallbackManagerForLLMRun
from pydantic import Field
class OpenRouterLLMClient:
def __init__(self, api_key: str, base_url: str = "https://openrouter.ai/api/v1"):
self.api_key = api_key
self.base_url = base_url
self.client = openai.OpenAI(
api_key=api_key,
base_url=base_url
)
self.async_client = openai.AsyncOpenAI(
api_key=api_key,
base_url=base_url
)
def generate_response(
self,
messages: List[Dict[str, str]],
provider: str,
model: str,
max_tokens: int = 4000,
temperature: float = 0.7,
top_p: float = 1.0,
**kwargs
) -> str:
"""Generate a response using the specified provider and model."""
# Format model name for OpenRouter (provider/model)
model_name = f"{provider}/{model}"
params = {
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
"top_p": top_p,
**kwargs
}
try:
response = self.client.chat.completions.create(**params)
return response.choices[0].message.content
except Exception as e:
raise Exception(f"Error generating response with {provider}/{model}: {str(e)}")
async def generate_response_async(
self,
messages: List[Dict[str, str]],
provider: str,
model: str,
max_tokens: int = 4000,
temperature: float = 0.7,
top_p: float = 1.0,
**kwargs
) -> str:
"""Async version of generate_response."""
model_name = f"{provider}/{model}"
params = {
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
"top_p": top_p,
**kwargs
}
try:
response = await self.async_client.chat.completions.create(**params)
return response.choices[0].message.content
except Exception as e:
raise Exception(f"Error generating response with {provider}/{model}: {str(e)}")
def stream_response(
self,
messages: List[Dict[str, str]],
provider: str,
model: str,
max_tokens: int = 4000,
temperature: float = 0.7,
top_p: float = 1.0,
**kwargs
):
"""Stream response from the specified provider and model."""
model_name = f"{provider}/{model}"
params = {
"model": model_name,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
"top_p": top_p,
"stream": True,
**kwargs
}
try:
stream = self.client.chat.completions.create(**params)
for chunk in stream:
if chunk.choices[0].delta.content is not None:
yield chunk.choices[0].delta.content
except Exception as e:
raise Exception(f"Error streaming response with {provider}/{model}: {str(e)}")
# Agent-compatible wrapper class
class LLMAgent:
def __init__(self, llm_client: OpenRouterLLMClient, default_provider: str, default_model: str):
self.llm_client = llm_client
self.default_provider = default_provider
self.default_model = default_model
self.conversation_history = []
def __call__(self, message: str, provider: Optional[str] = None, model: Optional[str] = None, **kwargs) -> str:
"""Make the agent callable - solves binding issues for agent frameworks."""
return self.chat(message, provider, model, **kwargs)
def chat(self, message: str, provider: Optional[str] = None, model: Optional[str] = None, **kwargs) -> str:
"""Chat with the LLM using the specified or default provider/model."""
provider = provider or self.default_provider
model = model or self.default_model
# Add user message to history
self.conversation_history.append({"role": "user", "content": message})
try:
response = self.llm_client.generate_response(
messages=self.conversation_history,
provider=provider,
model=model,
**kwargs
)
# Add assistant response to history
self.conversation_history.append({"role": "assistant", "content": response})
return response
except Exception as e:
return f"Error: {str(e)}"
async def chat_async(self, message: str, provider: Optional[str] = None, model: Optional[str] = None, **kwargs) -> str:
"""Async version of chat."""
provider = provider or self.default_provider
model = model or self.default_model
self.conversation_history.append({"role": "user", "content": message})
try:
response = await self.llm_client.generate_response_async(
messages=self.conversation_history,
provider=provider,
model=model,
**kwargs
)
self.conversation_history.append({"role": "assistant", "content": response})
return response
except Exception as e:
return f"Error: {str(e)}"
def stream_chat(self, message: str, provider: Optional[str] = None, model: Optional[str] = None, **kwargs):
"""Stream chat response."""
provider = provider or self.default_provider
model = model or self.default_model
self.conversation_history.append({"role": "user", "content": message})
try:
full_response = ""
for chunk in self.llm_client.stream_response(
messages=self.conversation_history,
provider=provider,
model=model,
**kwargs
):
full_response += chunk
yield chunk
# Add complete response to history
self.conversation_history.append({"role": "assistant", "content": full_response})
except Exception as e:
yield f"Error: {str(e)}"
def clear_history(self):
"""Clear conversation history."""
self.conversation_history = []
def get_history(self) -> List[Dict[str, str]]:
"""Get conversation history."""
return self.conversation_history.copy()
def set_system_message(self, system_message: str):
"""Set or update system message."""
# Remove existing system message if present
if self.conversation_history and self.conversation_history[0]["role"] == "system":
self.conversation_history.pop(0)
# Add new system message at the beginning
self.conversation_history.insert(0, {"role": "system", "content": system_message})
# --- SUBSET OF CODE TO REPLACE ---
# LangChain-compatible wrapper
class OpenRouterChatModel(BaseChatModel):
"""LangChain-compatible chat model wrapper for OpenRouter."""
llm_client: OpenRouterLLMClient = Field(...)
provider: str = Field(...)
model: str = Field(...)
max_tokens: int = Field(default=4000)
temperature: float = Field(default=0.7)
top_p: float = Field(default=1.0)
class Config:
arbitrary_types_allowed = True
@property
def _llm_type(self) -> str:
return "openrouter_chat"
def _convert_messages_to_openai_format(self, messages: List[BaseMessage]) -> List[Dict[str, str]]:
"""Convert LangChain messages to the OpenAI API format, correctly handling tool calls."""
openai_messages = []
for message in messages:
if isinstance(message, HumanMessage):
openai_messages.append({"role": "user", "content": message.content})
elif isinstance(message, AIMessage):
msg_dict = {"role": "assistant", "content": message.content or ""}
if message.tool_calls:
# This part now correctly receives a list of dicts and converts them
# to the format the API client expects for a new request.
msg_dict["tool_calls"] = [
{
"id": tc["id"],
"type": "function",
"function": {
"name": tc["name"],
"arguments": json.dumps(tc["args"]),
},
}
for tc in message.tool_calls
]
openai_messages.append(msg_dict)
elif isinstance(message, SystemMessage):
openai_messages.append({"role": "system", "content": message.content})
elif hasattr(message, "tool_call_id"): # Catches ToolMessage robustly
openai_messages.append({
"role": "tool",
"tool_call_id": message.tool_call_id,
"content": message.content,
})
else:
openai_messages.append({"role": "user", "content": str(message.content)})
return openai_messages
# This is a helper to centralize the logic for both sync and async calls
def _prepare_and_run_api(self, messages: List[BaseMessage], **kwargs: Any) -> openai.types.chat.ChatCompletion:
openai_messages = self._convert_messages_to_openai_format(messages)
# Merge instance parameters with kwargs
params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
}
# --- NEW: THIS IS THE CRITICAL CHANGE ---
# The agent passes tools via kwargs. We must grab them and add to the API params.
if "tools" in kwargs:
params["tools"] = kwargs["tools"]
# The client needs the raw response to check for tool calls, so we don't just return content
model_name = f"{self.provider}/{self.model}"
return self.llm_client.client.chat.completions.create(
model=model_name,
messages=openai_messages,
**params
)
async def _aprepare_and_run_api(self, messages: List[BaseMessage], **kwargs: Any) -> openai.types.chat.ChatCompletion:
openai_messages = self._convert_messages_to_openai_format(messages)
params = {
"max_tokens": kwargs.get("max_tokens", self.max_tokens),
"temperature": kwargs.get("temperature", self.temperature),
"top_p": kwargs.get("top_p", self.top_p),
}
if "tools" in kwargs:
params["tools"] = kwargs["tools"]
model_name = f"{self.provider}/{self.model}"
return await self.llm_client.async_client.chat.completions.create(
model=model_name,
messages=openai_messages,
**params
)
def _generate(
self,
messages: List[BaseMessage],
stop: Optional[List[str]] = None,
run_manager: Optional[CallbackManagerForLLMRun] = None,
**kwargs: Any,
) -> ChatResult:
"""Generate chat response."""
try:
response = self._prepare_and_run_api(messages, **kwargs)
response_message = response.choices[0].message
# THIS IS THE NEW, CRITICAL PARSING STEP
parsed_tool_calls = []
if response_message.tool_calls:
for tc in response_message.tool_calls:
parsed_tool_calls.append({
"name": tc.function.name,
"args": json.loads(tc.function.arguments),
"id": tc.id,
})
message = AIMessage(
content=response_message.content or "",
tool_calls=parsed_tool_calls # Pass the parsed list of dicts
)
generation = ChatGeneration(message=message)
return ChatResult(generations=[generation])
except Exception as e:
raise Exception(f"Error in OpenRouter chat generation: {str(e)}")
async def _agenerate(
self,
messages: List[BaseMessage],
stop: Optional[List[str]] = None,
run_manager: Optional[CallbackManagerForLLMRun] = None,
**kwargs: Any,
) -> ChatResult:
"""Async generate chat response."""
try:
response = await self._aprepare_and_run_api(messages, **kwargs)
response_message = response.choices[0].message
# THIS IS THE NEW, CRITICAL PARSING STEP
parsed_tool_calls = []
if response_message.tool_calls:
for tc in response_message.tool_calls:
parsed_tool_calls.append({
"name": tc.function.name,
"args": json.loads(tc.function.arguments),
"id": tc.id,
})
message = AIMessage(
content=response_message.content or "",
tool_calls=parsed_tool_calls # Pass the parsed list of dicts
)
generation = ChatGeneration(message=message)
return ChatResult(generations=[generation])
except Exception as e:
raise Exception(f"Error in OpenRouter async chat generation: {str(e)}")
# Convenience function to create LangChain-compatible LLM
def create_openrouter_llm(api_key: str, provider: str, model: str, **kwargs) -> OpenRouterChatModel:
"""Create a LangChain-compatible OpenRouter LLM."""
llm_client = OpenRouterLLMClient(api_key)
return OpenRouterChatModel(
llm_client=llm_client,
provider=provider,
model=model,
**kwargs
)