| from langchain.agents import load_tools |
| from langchain.agents import initialize_agent |
| from langchain.agents import AgentType |
| from langchain.chat_models import AzureChatOpenAI |
| from langchain.llms import OpenAI |
| from langchain.chains.conversation.memory import ConversationBufferWindowMemory |
|
|
| import os |
|
|
|
|
| OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") |
| OPENAI_API_BASE = os.getenv("OPENAI_API_BASE") |
| |
| llm = AzureChatOpenAI(deployment_name="bitservice_chat_35",openai_api_base=OPENAI_API_BASE,openai_api_key=OPENAI_API_KEY,openai_api_version="2023-03-15-preview",model_name="gpt-3.5-turbo") |
|
|
|
|
| import torch |
| from transformers import BlipProcessor, BlipForConditionalGeneration |
|
|
| image_to_text_model = "Salesforce/blip-image-captioning-large" |
| device = 'cuda' if torch.cuda.is_available() else 'cpu' |
|
|
| processor = BlipProcessor.from_pretrained(image_to_text_model) |
| model = BlipForConditionalGeneration.from_pretrained(image_to_text_model).to(device) |
|
|
| from transformers.models.oneformer.modeling_oneformer import OneFormerModelOutput |
| import requests |
| from PIL import Image |
|
|
| def describeImage(image_url): |
| image_object = Image.open(image_url).convert('RGB') |
| |
| inputs = processor(image_object, return_tensors="pt").to(device) |
| outputs = model.generate(**inputs) |
| return processor.decode(outputs[0], skip_special_tokens=True) |
|
|
|
|
| from langchain.tools import BaseTool |
|
|
| class DescribeImageTool(BaseTool): |
| name = "Describe Image Tool" |
| description = 'use this tool to describe an image.' |
|
|
| def _run(self, url: str): |
| description = describeImage(url) |
| return description |
| |
| def _arun(self, query: str): |
| raise NotImplementedError("Async operation not supported yet") |
|
|
| tools = [DescribeImageTool()] |
|
|
|
|
| agent = initialize_agent( |
| agent='chat-conversational-react-description', |
| tools=tools, |
| llm=llm, |
| verbose=True, |
| max_iterations=3, |
| early_stopping_method='generate', |
| memory=ConversationBufferWindowMemory( |
| memory_key='chat_history', |
| k=5, |
| return_messages=True |
| ) |
| ) |
|
|
| from langchain.chains import LLMChain |
| from langchain.prompts import PromptTemplate |
| def enToChinese(english): |
| |
| |
| pp = "Please translate the following sentence from English to Chinese：{english}" |
| prompt = PromptTemplate( |
| input_variables=["english"], |
| template=pp |
| ) |
| llchain=LLMChain(llm=llm,prompt=prompt) |
| return llchain.run(english) |
| |
|
|
| def chToEnglish(chinese): |
| |
| |
| pp = "Please translate the following sentence from Chinese to English：{chinese}" |
| prompt = PromptTemplate( |
| input_variables=["chinese"], |
| template=pp |
| ) |
| llchain=LLMChain(llm=llm,prompt=prompt) |
| return llchain.run(chinese) |
|
|
| import gradio as gr |
| def segment(image,text): |
| |
| print(image) |
| image_url = image |
| |
| print(text) |
| return enToChinese(agent(f"{text}:\n{image_url}").get('output')) |
|
|
| demo = gr.Interface( |
| fn=segment, |
| inputs=[ |
| gr.Image(type="filepath",shape=(200, 200),label="请选择一张图片"), |
| gr.components.Textbox(label="请输入问题"), |
| ], |
| outputs=[gr.components.Textbox(label="答案",lines=4)]) |
| demo.launch() |
|
|
|
|
