        image_with_box = self.plot_boxes_to_image(image_pil, pred_dict)[0]

        updated_image_path = get_new_image_name(image_path, func_name="detect-something")
        updated_image = image_with_box.resize(size)
        updated_image.save(updated_image_path)
        print(
            f"\nProcessed ObejectDetecting, Input Image: {image_path}, Object to be Detect {det_prompt}, "
            f"Output Image: {updated_image_path}")
        return updated_image_path

class Inpainting:
    def __init__(self, device):
        self.device = device
        self.revision = 'fp16' if 'cuda' in self.device else None
        self.torch_dtype = torch.float16 if 'cuda' in self.device else torch.float32

        self.inpaint = StableDiffusionInpaintPipeline.from_pretrained(
            "runwayml/stable-diffusion-inpainting", revision=self.revision, torch_dtype=self.torch_dtype,safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker')).to(device)
    def __call__(self, prompt, image, mask_image, height=512, width=512, num_inference_steps=50):
        update_image = self.inpaint(prompt=prompt, image=image.resize((width, height)),
                                     mask_image=mask_image.resize((width, height)), height=height, width=width, num_inference_steps=num_inference_steps).images[0]
        return update_image

class InfinityOutPainting:
    template_model = True # Add this line to show this is a template model.
    def __init__(self, ImageCaptioning, Inpainting, VisualQuestionAnswering):
        self.llm = OpenAI(temperature=0)
        self.ImageCaption = ImageCaptioning
        self.inpaint = Inpainting
        self.ImageVQA = VisualQuestionAnswering
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
                        'fewer digits, cropped, worst quality, low quality'

    def get_BLIP_vqa(self, image, question):
        inputs = self.ImageVQA.processor(image, question, return_tensors="pt").to(self.ImageVQA.device,
                                                                                self.ImageVQA.torch_dtype)
        out = self.ImageVQA.model.generate(**inputs)
        answer = self.ImageVQA.processor.decode(out[0], skip_special_tokens=True)
        print(f"\nProcessed VisualQuestionAnswering, Input Question: {question}, Output Answer: {answer}")
        return answer


        self.torch_dtype = torch.float16 if 'cuda' in self.device else torch.float32

        self.inpaint = StableDiffusionInpaintPipeline.from_pretrained(
            "runwayml/stable-diffusion-inpainting", revision=self.revision, torch_dtype=self.torch_dtype,safety_checker=StableDiffusionSafetyChecker.from_pretrained('CompVis/stable-diffusion-safety-checker')).to(device)
    def __call__(self, prompt, image, mask_image, height=512, width=512, num_inference_steps=50):
        update_image = self.inpaint(prompt=prompt, image=image.resize((width, height)),
                                     mask_image=mask_image.resize((width, height)), height=height, width=width, num_inference_steps=num_inference_steps).images[0]
        return update_image

class InfinityOutPainting:
    template_model = True # Add this line to show this is a template model.
    def __init__(self, ImageCaptioning, Inpainting, VisualQuestionAnswering):
        self.llm = OpenAI(temperature=0)
        self.ImageCaption = ImageCaptioning
        self.inpaint = Inpainting
        self.ImageVQA = VisualQuestionAnswering
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
                        'fewer digits, cropped, worst quality, low quality'

    def get_BLIP_vqa(self, image, question):
        inputs = self.ImageVQA.processor(image, question, return_tensors="pt").to(self.ImageVQA.device,
                                                                                self.ImageVQA.torch_dtype)
        out = self.ImageVQA.model.generate(**inputs)
        answer = self.ImageVQA.processor.decode(out[0], skip_special_tokens=True)
        print(f"\nProcessed VisualQuestionAnswering, Input Question: {question}, Output Answer: {answer}")
        return answer

    def get_BLIP_caption(self, image):
        inputs = self.ImageCaption.processor(image, return_tensors="pt").to(self.ImageCaption.device,
                                                                                self.ImageCaption.torch_dtype)
        out = self.ImageCaption.model.generate(**inputs)
        BLIP_caption = self.ImageCaption.processor.decode(out[0], skip_special_tokens=True)
        return BLIP_caption

    def check_prompt(self, prompt):
        check = f"Here is a paragraph with adjectives. " \
                f"{prompt} " \
                f"Please change all plural forms in the adjectives to singular forms. "
        return self.llm(check)

    def get_imagine_caption(self, image, imagine):
        BLIP_caption = self.get_BLIP_caption(image)
        background_color = self.get_BLIP_vqa(image, 'what is the background color of this image')
        style = self.get_BLIP_vqa(image, 'what is the style of this image')
        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \


class InfinityOutPainting:
    template_model = True # Add this line to show this is a template model.
    def __init__(self, ImageCaptioning, Inpainting, VisualQuestionAnswering):
        self.llm = OpenAI(temperature=0)
        self.ImageCaption = ImageCaptioning
        self.inpaint = Inpainting
        self.ImageVQA = VisualQuestionAnswering
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
                        'fewer digits, cropped, worst quality, low quality'

    def get_BLIP_vqa(self, image, question):
        inputs = self.ImageVQA.processor(image, question, return_tensors="pt").to(self.ImageVQA.device,
                                                                                self.ImageVQA.torch_dtype)
        out = self.ImageVQA.model.generate(**inputs)
        answer = self.ImageVQA.processor.decode(out[0], skip_special_tokens=True)
        print(f"\nProcessed VisualQuestionAnswering, Input Question: {question}, Output Answer: {answer}")
        return answer

    def get_BLIP_caption(self, image):
        inputs = self.ImageCaption.processor(image, return_tensors="pt").to(self.ImageCaption.device,
                                                                                self.ImageCaption.torch_dtype)
        out = self.ImageCaption.model.generate(**inputs)
        BLIP_caption = self.ImageCaption.processor.decode(out[0], skip_special_tokens=True)
        return BLIP_caption

    def check_prompt(self, prompt):
        check = f"Here is a paragraph with adjectives. " \
                f"{prompt} " \
                f"Please change all plural forms in the adjectives to singular forms. "
        return self.llm(check)

    def get_imagine_caption(self, image, imagine):
        BLIP_caption = self.get_BLIP_caption(image)
        background_color = self.get_BLIP_vqa(image, 'what is the background color of this image')
        style = self.get_BLIP_vqa(image, 'what is the style of this image')
        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \
                         f"please imagine the complete painting and describe it" \
                         f"you should consider the background color is {background_color}, the style is {style}" \
                         f"You should make the painting as vivid and realistic as possible" \
                         f"You can not use words like painting or picture" \
                         f"and you should use no more than 50 words to describe it"
        caption = self.llm(imagine_prompt) if imagine else BLIP_caption
        caption = self.check_prompt(caption)


        self.ImageVQA = VisualQuestionAnswering
        self.a_prompt = 'best quality, extremely detailed'
        self.n_prompt = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, ' \
                        'fewer digits, cropped, worst quality, low quality'

    def get_BLIP_vqa(self, image, question):
        inputs = self.ImageVQA.processor(image, question, return_tensors="pt").to(self.ImageVQA.device,
                                                                                self.ImageVQA.torch_dtype)
        out = self.ImageVQA.model.generate(**inputs)
        answer = self.ImageVQA.processor.decode(out[0], skip_special_tokens=True)
        print(f"\nProcessed VisualQuestionAnswering, Input Question: {question}, Output Answer: {answer}")
        return answer

    def get_BLIP_caption(self, image):
        inputs = self.ImageCaption.processor(image, return_tensors="pt").to(self.ImageCaption.device,
                                                                                self.ImageCaption.torch_dtype)
        out = self.ImageCaption.model.generate(**inputs)
        BLIP_caption = self.ImageCaption.processor.decode(out[0], skip_special_tokens=True)
        return BLIP_caption

    def check_prompt(self, prompt):
        check = f"Here is a paragraph with adjectives. " \
                f"{prompt} " \
                f"Please change all plural forms in the adjectives to singular forms. "
        return self.llm(check)

    def get_imagine_caption(self, image, imagine):
        BLIP_caption = self.get_BLIP_caption(image)
        background_color = self.get_BLIP_vqa(image, 'what is the background color of this image')
        style = self.get_BLIP_vqa(image, 'what is the style of this image')
        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \
                         f"please imagine the complete painting and describe it" \
                         f"you should consider the background color is {background_color}, the style is {style}" \
                         f"You should make the painting as vivid and realistic as possible" \
                         f"You can not use words like painting or picture" \
                         f"and you should use no more than 50 words to describe it"
        caption = self.llm(imagine_prompt) if imagine else BLIP_caption
        caption = self.check_prompt(caption)
        print(f'BLIP observation: {BLIP_caption}, ChatGPT imagine to {caption}') if imagine else print(
            f'Prompt: {caption}')
        return caption

    def resize_image(self, image, max_size=1000000, multiple=8):
        aspect_ratio = image.size[0] / image.size[1]


        inputs = self.ImageVQA.processor(image, question, return_tensors="pt").to(self.ImageVQA.device,
                                                                                self.ImageVQA.torch_dtype)
        out = self.ImageVQA.model.generate(**inputs)
        answer = self.ImageVQA.processor.decode(out[0], skip_special_tokens=True)
        print(f"\nProcessed VisualQuestionAnswering, Input Question: {question}, Output Answer: {answer}")
        return answer

    def get_BLIP_caption(self, image):
        inputs = self.ImageCaption.processor(image, return_tensors="pt").to(self.ImageCaption.device,
                                                                                self.ImageCaption.torch_dtype)
        out = self.ImageCaption.model.generate(**inputs)
        BLIP_caption = self.ImageCaption.processor.decode(out[0], skip_special_tokens=True)
        return BLIP_caption

    def check_prompt(self, prompt):
        check = f"Here is a paragraph with adjectives. " \
                f"{prompt} " \
                f"Please change all plural forms in the adjectives to singular forms. "
        return self.llm(check)

    def get_imagine_caption(self, image, imagine):
        BLIP_caption = self.get_BLIP_caption(image)
        background_color = self.get_BLIP_vqa(image, 'what is the background color of this image')
        style = self.get_BLIP_vqa(image, 'what is the style of this image')
        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \
                         f"please imagine the complete painting and describe it" \
                         f"you should consider the background color is {background_color}, the style is {style}" \
                         f"You should make the painting as vivid and realistic as possible" \
                         f"You can not use words like painting or picture" \
                         f"and you should use no more than 50 words to describe it"
        caption = self.llm(imagine_prompt) if imagine else BLIP_caption
        caption = self.check_prompt(caption)
        print(f'BLIP observation: {BLIP_caption}, ChatGPT imagine to {caption}') if imagine else print(
            f'Prompt: {caption}')
        return caption

    def resize_image(self, image, max_size=1000000, multiple=8):
        aspect_ratio = image.size[0] / image.size[1]
        new_width = int(math.sqrt(max_size * aspect_ratio))
        new_height = int(new_width / aspect_ratio)
        new_width, new_height = new_width - (new_width % multiple), new_height - (new_height % multiple)
        return image.resize((new_width, new_height))

    def dowhile(self, original_img, tosize, expand_ratio, imagine, usr_prompt):
        old_img = original_img
        while (old_img.size != tosize):
            prompt = self.check_prompt(usr_prompt) if usr_prompt else self.get_imagine_caption(old_img, imagine)
            crop_w = 15 if old_img.size[0] != tosize[0] else 0
            crop_h = 15 if old_img.size[1] != tosize[1] else 0
            old_img = ImageOps.crop(old_img, (crop_w, crop_h, crop_w, crop_h))
            temp_canvas_size = (expand_ratio * old_img.width if expand_ratio * old_img.width < tosize[0] else tosize[0],
                                expand_ratio * old_img.height if expand_ratio * old_img.height < tosize[1] else tosize[
                                    1])
            temp_canvas, temp_mask = Image.new("RGB", temp_canvas_size, color="white"), Image.new("L", temp_canvas_size,
                                                                                color="white")


                f"Please change all plural forms in the adjectives to singular forms. "
        return self.llm(check)

    def get_imagine_caption(self, image, imagine):
        BLIP_caption = self.get_BLIP_caption(image)
        background_color = self.get_BLIP_vqa(image, 'what is the background color of this image')
        style = self.get_BLIP_vqa(image, 'what is the style of this image')
        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \
                         f"please imagine the complete painting and describe it" \
                         f"you should consider the background color is {background_color}, the style is {style}" \
                         f"You should make the painting as vivid and realistic as possible" \
                         f"You can not use words like painting or picture" \
                         f"and you should use no more than 50 words to describe it"
        caption = self.llm(imagine_prompt) if imagine else BLIP_caption
        caption = self.check_prompt(caption)
        print(f'BLIP observation: {BLIP_caption}, ChatGPT imagine to {caption}') if imagine else print(
            f'Prompt: {caption}')
        return caption

    def resize_image(self, image, max_size=1000000, multiple=8):
        aspect_ratio = image.size[0] / image.size[1]
        new_width = int(math.sqrt(max_size * aspect_ratio))
        new_height = int(new_width / aspect_ratio)
        new_width, new_height = new_width - (new_width % multiple), new_height - (new_height % multiple)
        return image.resize((new_width, new_height))

    def dowhile(self, original_img, tosize, expand_ratio, imagine, usr_prompt):
        old_img = original_img
        while (old_img.size != tosize):
            prompt = self.check_prompt(usr_prompt) if usr_prompt else self.get_imagine_caption(old_img, imagine)
            crop_w = 15 if old_img.size[0] != tosize[0] else 0
            crop_h = 15 if old_img.size[1] != tosize[1] else 0
            old_img = ImageOps.crop(old_img, (crop_w, crop_h, crop_w, crop_h))
            temp_canvas_size = (expand_ratio * old_img.width if expand_ratio * old_img.width < tosize[0] else tosize[0],
                                expand_ratio * old_img.height if expand_ratio * old_img.height < tosize[1] else tosize[
                                    1])
            temp_canvas, temp_mask = Image.new("RGB", temp_canvas_size, color="white"), Image.new("L", temp_canvas_size,
                                                                                color="white")
            x, y = (temp_canvas.width - old_img.width) // 2, (temp_canvas.height - old_img.height) // 2
            temp_canvas.paste(old_img, (x, y))
            temp_mask.paste(0, (x, y, x + old_img.width, y + old_img.height))
            resized_temp_canvas, resized_temp_mask = self.resize_image(temp_canvas), self.resize_image(temp_mask)
            image = self.inpaint(prompt=prompt, image=resized_temp_canvas, mask_image=resized_temp_mask,
                                              height=resized_temp_canvas.height, width=resized_temp_canvas.width,
                                              num_inference_steps=50).resize(


dowhile

        imagine_prompt = f"let's pretend you are an excellent painter and now " \
                         f"there is an incomplete painting with {BLIP_caption} in the center, " \
                         f"please imagine the complete painting and describe it" \
                         f"you should consider the background color is {background_color}, the style is {style}" \
                         f"You should make the painting as vivid and realistic as possible" \
                         f"You can not use words like painting or picture" \
                         f"and you should use no more than 50 words to describe it"
        caption = self.llm(imagine_prompt) if imagine else BLIP_caption
        caption = self.check_prompt(caption)
        print(f'BLIP observation: {BLIP_caption}, ChatGPT imagine to {caption}') if imagine else print(
            f'Prompt: {caption}')
        return caption

    def resize_image(self, image, max_size=1000000, multiple=8):
        aspect_ratio = image.size[0] / image.size[1]
        new_width = int(math.sqrt(max_size * aspect_ratio))
        new_height = int(new_width / aspect_ratio)
        new_width, new_height = new_width - (new_width % multiple), new_height - (new_height % multiple)
        return image.resize((new_width, new_height))

    def dowhile(self, original_img, tosize, expand_ratio, imagine, usr_prompt):
        old_img = original_img
        while (old_img.size != tosize):
            prompt = self.check_prompt(usr_prompt) if usr_prompt else self.get_imagine_caption(old_img, imagine)
            crop_w = 15 if old_img.size[0] != tosize[0] else 0
            crop_h = 15 if old_img.size[1] != tosize[1] else 0
            old_img = ImageOps.crop(old_img, (crop_w, crop_h, crop_w, crop_h))
            temp_canvas_size = (expand_ratio * old_img.width if expand_ratio * old_img.width < tosize[0] else tosize[0],
                                expand_ratio * old_img.height if expand_ratio * old_img.height < tosize[1] else tosize[
                                    1])
            temp_canvas, temp_mask = Image.new("RGB", temp_canvas_size, color="white"), Image.new("L", temp_canvas_size,
                                                                                color="white")
            x, y = (temp_canvas.width - old_img.width) // 2, (temp_canvas.height - old_img.height) // 2
            temp_canvas.paste(old_img, (x, y))
            temp_mask.paste(0, (x, y, x + old_img.width, y + old_img.height))
            resized_temp_canvas, resized_temp_mask = self.resize_image(temp_canvas), self.resize_image(temp_mask)
            image = self.inpaint(prompt=prompt, image=resized_temp_canvas, mask_image=resized_temp_mask,
                                              height=resized_temp_canvas.height, width=resized_temp_canvas.width,
                                              num_inference_steps=50).resize(
                (temp_canvas.width, temp_canvas.height), Image.ANTIALIAS)
            image = blend_gt2pt(old_img, image)
            old_img = image
        return old_img

    @prompts(name="Extend An Image",
             description="useful when you need to extend an image into a larger image."
                         "like: extend the image into a resolution of 2048x1024, extend the image into 2048x1024. "
                         "The input to this tool should be a comma separated string of two, representing the image_path and the resolution of widthxheight")
    def inference(self, inputs):
        image_path, resolution = inputs.split(',')
        width, height = resolution.split('x')
        tosize = (int(width), int(height))
        image = Image.open(image_path)
        image = ImageOps.crop(image, (10, 10, 10, 10))
        out_painted_image = self.dowhile(image, tosize, 4, True, False)
        updated_image_path = get_new_image_name(image_path, func_name="outpainting")
        out_painted_image.save(updated_image_path)
        print(f"\nProcessed InfinityOutPainting, Input Image: {image_path}, Input Resolution: {resolution}, "
              f"Output Image: {updated_image_path}")
        return updated_image_path


    @prompts(name="Segment the given object",
            description="useful when you only want to segment the certain objects in the picture"
                        "according to the given text"
                        "like: segment the cat,"
                        "or can you segment an obeject for me"
                        "The input to this tool should be a comma separated string of two, "
                        "representing the image_path, the text description of the object to be found")
    def inference(self, inputs):
        image_path, det_prompt = inputs.split(",")
        print(f"image_path={image_path}, text_prompt={det_prompt}")
        image_pil, image = self.grounding.load_image(image_path)

        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, det_prompt)
        updated_image_path = self.sam.segment_image_with_boxes(image_pil,image_path,boxes_filt,pred_phrases)
        print(
            f"\nProcessed ObejectSegmenting, Input Image: {image_path}, Object to be Segment {det_prompt}, "
            f"Output Image: {updated_image_path}")
        return updated_image_path

    def merge_masks(self, masks):
        '''
            Args:
                mask (numpy.ndarray): shape N x 1 x H x W
            Outputs:
                new_mask (numpy.ndarray): shape H x W
        '''
        if type(masks) == torch.Tensor:
            x = masks
        elif type(masks) == np.ndarray:
            x = torch.tensor(masks,dtype=int)
        else:
            raise TypeError("the type of the input masks must be numpy.ndarray or torch.tensor")
        x = x.squeeze(dim=1)
        value, _ = x.max(dim=0)
        new_mask = value.cpu().numpy()
        new_mask.astype(np.uint8)
        return new_mask

    def get_mask(self, image_path, text_prompt):

        print(f"image_path={image_path}, text_prompt={text_prompt}")
        # image_pil (PIL.Image.Image) -> size: W x H
        # image (numpy.ndarray) -> H x W x 3
        image_pil, image = self.grounding.load_image(image_path)

        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, text_prompt)
        image = cv2.imread(image_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        self.sam.sam_predictor.set_image(image)

        # masks (torch.tensor) -> N x 1 x H x W
        masks = self.sam.get_mask_with_boxes(image_pil, image, boxes_filt)

        # merged_mask -> H x W
        merged_mask = self.merge_masks(masks)
        # draw output image


             description="useful when you want to extract the object or remove the background,"
                         "the input should be a string image_path"
                                )
    def inference(self, image_path):
        '''
            given a image, return the picture only contains the extracted main object
        '''
        updated_image_path = None

        mask = self.get_mask(image_path)

        image = Image.open(image_path)
        mask = Image.fromarray(mask)
        image.putalpha(mask)

        updated_image_path = get_new_image_name(image_path, func_name="detect-something")
        image.save(updated_image_path)

        return updated_image_path

    def get_mask(self, image_path):
        '''
            Description:
                given an image path, return the mask of the main object.
            Args:
                image_path (string): the file path of the image
            Outputs:
                mask (numpy.ndarray): H x W
        '''
        vqa_input = f"{image_path}, what is the main object in the image?"
        text_prompt = self.vqa.inference(vqa_input)

        mask = self.obj_segmenting.get_mask(image_path,text_prompt)

        return mask

class ConversationBot:
    def __init__(self, load_dict):
        # load_dict = {'VisualQuestionAnswering':'cuda:0', 'ImageCaptioning':'cuda:1',...}
        print(f"Initializing VisualChatGPT, load_dict={load_dict}")
        if 'ImageCaptioning' not in load_dict:
            raise ValueError("You have to load ImageCaptioning as a basic function for VisualChatGPT")

        self.models = {}
        # Load Basic Foundation Models
        for class_name, device in load_dict.items():
            self.models[class_name] = globals()[class_name](device=device)

        # Load Template Foundation Models
        for class_name, module in globals().items():
            if getattr(module, 'template_model', False):
                template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'}
                loaded_names = set([type(e).__name__ for e in self.models.values()])
                if template_required_names.issubset(loaded_names):


        merged_mask = self.merge_masks(masks)
        # draw output image

        for mask in masks:
            image = self.sam.show_mask(mask[0].cpu().numpy(), image, random_color=True, transparency=0.3)

        merged_mask_image = Image.fromarray(merged_mask)

        return merged_mask

class ImageEditing:
    template_model = True
    def __init__(self, Text2Box:Text2Box, Segmenting:Segmenting, Inpainting:Inpainting):
        print(f"Initializing ImageEditing")
        self.sam = Segmenting
        self.grounding = Text2Box
        self.inpaint = Inpainting

    def pad_edge(self,mask,padding):
        #mask Tensor [H,W]
        mask = mask.numpy()
        true_indices = np.argwhere(mask)
        mask_array = np.zeros_like(mask, dtype=bool)
        for idx in true_indices:
            padded_slice = tuple(slice(max(0, i - padding), i + padding + 1) for i in idx)
            mask_array[padded_slice] = True
        new_mask = (mask_array * 255).astype(np.uint8)
        #new_mask
        return new_mask

    @prompts(name="Remove Something From The Photo",
             description="useful when you want to remove and object or something from the photo "
                         "from its description or location. "
                         "The input to this tool should be a comma separated string of two, "
                         "representing the image_path and the object need to be removed. ")
    def inference_remove(self, inputs):
        image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        return self.inference_replace_sam(f"{image_path},{to_be_removed_txt},background")

    @prompts(name="Replace Something From The Photo",
            description="useful when you want to replace an object from the object description or "
                        "location with another object from its description. "
                        "The input to this tool should be a comma separated string of three, "
                        "representing the image_path, the object to be replaced, the object to be replaced with ")
    def inference_replace_sam(self,inputs):
        image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",")

        print(f"image_path={image_path}, to_be_replaced_txt={to_be_replaced_txt}")
        image_pil, image = self.grounding.load_image(image_path)


        self.grounding = Text2Box
        self.inpaint = Inpainting

    def pad_edge(self,mask,padding):
        #mask Tensor [H,W]
        mask = mask.numpy()
        true_indices = np.argwhere(mask)
        mask_array = np.zeros_like(mask, dtype=bool)
        for idx in true_indices:
            padded_slice = tuple(slice(max(0, i - padding), i + padding + 1) for i in idx)
            mask_array[padded_slice] = True
        new_mask = (mask_array * 255).astype(np.uint8)
        #new_mask
        return new_mask

    @prompts(name="Remove Something From The Photo",
             description="useful when you want to remove and object or something from the photo "
                         "from its description or location. "
                         "The input to this tool should be a comma separated string of two, "
                         "representing the image_path and the object need to be removed. ")
    def inference_remove(self, inputs):
        image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        return self.inference_replace_sam(f"{image_path},{to_be_removed_txt},background")

    @prompts(name="Replace Something From The Photo",
            description="useful when you want to replace an object from the object description or "
                        "location with another object from its description. "
                        "The input to this tool should be a comma separated string of three, "
                        "representing the image_path, the object to be replaced, the object to be replaced with ")
    def inference_replace_sam(self,inputs):
        image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",")

        print(f"image_path={image_path}, to_be_replaced_txt={to_be_replaced_txt}")
        image_pil, image = self.grounding.load_image(image_path)
        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, to_be_replaced_txt)
        image = cv2.imread(image_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        self.sam.sam_predictor.set_image(image)
        masks = self.sam.get_mask_with_boxes(image_pil, image, boxes_filt)
        mask = torch.sum(masks, dim=0).unsqueeze(0)
        mask = torch.where(mask > 0, True, False)
        mask = mask.squeeze(0).squeeze(0).cpu() #tensor


            padded_slice = tuple(slice(max(0, i - padding), i + padding + 1) for i in idx)
            mask_array[padded_slice] = True
        new_mask = (mask_array * 255).astype(np.uint8)
        #new_mask
        return new_mask

    @prompts(name="Remove Something From The Photo",
             description="useful when you want to remove and object or something from the photo "
                         "from its description or location. "
                         "The input to this tool should be a comma separated string of two, "
                         "representing the image_path and the object need to be removed. ")
    def inference_remove(self, inputs):
        image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
        return self.inference_replace_sam(f"{image_path},{to_be_removed_txt},background")

    @prompts(name="Replace Something From The Photo",
            description="useful when you want to replace an object from the object description or "
                        "location with another object from its description. "
                        "The input to this tool should be a comma separated string of three, "
                        "representing the image_path, the object to be replaced, the object to be replaced with ")
    def inference_replace_sam(self,inputs):
        image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",")

        print(f"image_path={image_path}, to_be_replaced_txt={to_be_replaced_txt}")
        image_pil, image = self.grounding.load_image(image_path)
        boxes_filt, pred_phrases = self.grounding.get_grounding_boxes(image, to_be_replaced_txt)
        image = cv2.imread(image_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        self.sam.sam_predictor.set_image(image)
        masks = self.sam.get_mask_with_boxes(image_pil, image, boxes_filt)
        mask = torch.sum(masks, dim=0).unsqueeze(0)
        mask = torch.where(mask > 0, True, False)
        mask = mask.squeeze(0).squeeze(0).cpu() #tensor

        mask = self.pad_edge(mask,padding=20) #numpy
        mask_image = Image.fromarray(mask)

        updated_image = self.inpaint(prompt=replace_with_txt, image=image_pil,
                                     mask_image=mask_image)
        updated_image_path = get_new_image_name(image_path, func_name="replace-something")
        updated_image = updated_image.resize(image_pil.size)
        updated_image.save(updated_image_path)
        print(
            f"\nProcessed ImageEditing, Input Image: {image_path}, Replace {to_be_replaced_txt} to {replace_with_txt}, "
            f"Output Image: {updated_image_path}")
        return updated_image_path

class BackgroundRemoving:
    '''
        using to remove the background of the given picture
    '''
    template_model = True
    def __init__(self,VisualQuestionAnswering:VisualQuestionAnswering, Text2Box:Text2Box, Segmenting:Segmenting):
        self.vqa = VisualQuestionAnswering
        self.obj_segmenting = ObjectSegmenting(Text2Box,Segmenting)

    @prompts(name="Remove the background",
             description="useful when you want to extract the object or remove the background,"
                         "the input should be a string image_path"
                                )
    def inference(self, image_path):
        '''
            given a image, return the picture only contains the extracted main object
        '''
        updated_image_path = None


        # Load Template Foundation Models
        for class_name, module in globals().items():
            if getattr(module, 'template_model', False):
                template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'}
                loaded_names = set([type(e).__name__ for e in self.models.values()])
                if template_required_names.issubset(loaded_names):
                    self.models[class_name] = globals()[class_name](
                        **{name: self.models[name] for name in template_required_names})

        print(f"All the Available Functions: {self.models}")

        self.tools = []
        for instance in self.models.values():
            for e in dir(instance):
                if e.startswith('inference'):
                    func = getattr(instance, e)
                    self.tools.append(Tool(name=func.name, description=func.description, func=func))
        self.llm = OpenAI(temperature=0)
        self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')

    def init_agent(self, lang):
        self.memory.clear() #clear previous history
        if lang=='English':
            PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS, VISUAL_CHATGPT_SUFFIX
            place = "Enter text and press enter, or upload an image"
            label_clear = "Clear"
        else:
            PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX_CN, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS_CN, VISUAL_CHATGPT_SUFFIX_CN
            place = "输入文字并回车，或者上传图片"
            label_clear = "清除"
        self.agent = initialize_agent(
            self.tools,
            self.llm,
            agent="conversational-react-description",
            verbose=True,
            memory=self.memory,
            return_intermediate_steps=True,
            agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
                          'suffix': SUFFIX}, )
        return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear)

    def run_text(self, text, state):
        self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500)
        res = self.agent({"input": text.strip()})
        res['output'] = res['output'].replace("\\", "/")
        response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output'])
        state = state + [(text, response)]
        print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
              f"Current Memory: {self.agent.memory.buffer}")
        return state, state

    def run_image(self, image, state, txt, lang):
        image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
        print("======>Auto Resize Image...")
        img = Image.open(image.name)
        width, height = img.size
        ratio = min(512 / width, 512 / height)
        width_new, height_new = (round(width * ratio), round(height * ratio))
        width_new = int(np.round(width_new / 64.0)) * 64
        height_new = int(np.round(height_new / 64.0)) * 64


        self.memory.clear() #clear previous history
        if lang=='English':
            PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS, VISUAL_CHATGPT_SUFFIX
            place = "Enter text and press enter, or upload an image"
            label_clear = "Clear"
        else:
            PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = VISUAL_CHATGPT_PREFIX_CN, VISUAL_CHATGPT_FORMAT_INSTRUCTIONS_CN, VISUAL_CHATGPT_SUFFIX_CN
            place = "输入文字并回车，或者上传图片"
            label_clear = "清除"
        self.agent = initialize_agent(
            self.tools,
            self.llm,
            agent="conversational-react-description",
            verbose=True,
            memory=self.memory,
            return_intermediate_steps=True,
            agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
                          'suffix': SUFFIX}, )
        return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear)

    def run_text(self, text, state):
        self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500)
        res = self.agent({"input": text.strip()})
        res['output'] = res['output'].replace("\\", "/")
        response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output'])
        state = state + [(text, response)]
        print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
              f"Current Memory: {self.agent.memory.buffer}")
        return state, state

    def run_image(self, image, state, txt, lang):
        image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
        print("======>Auto Resize Image...")
        img = Image.open(image.name)
        width, height = img.size
        ratio = min(512 / width, 512 / height)
        width_new, height_new = (round(width * ratio), round(height * ratio))
        width_new = int(np.round(width_new / 64.0)) * 64
        height_new = int(np.round(height_new / 64.0)) * 64
        img = img.resize((width_new, height_new))
        img = img.convert('RGB')
        img.save(image_filename, "PNG")
        print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
        description = self.models['ImageCaptioning'].inference(image_filename)
        if lang == 'Chinese':
            Human_prompt = f'\nHuman: 提供一张名为 {image_filename}的图片。它的描述是: {description}。 这些信息帮助你理解这个图像，但是你应该使用工具来完成下面的任务，而不是直接从我的描述中想象。 如果你明白了, 说 \"收到\". \n'
            AI_prompt = "收到。  "
        else:
            Human_prompt = f'\nHuman: provide a figure named {image_filename}. The description is: {description}. This information helps you to understand this image, but you should use tools to finish following tasks, rather than directly imagine from my description. If you understand, say \"Received\". \n'


            self.tools,
            self.llm,
            agent="conversational-react-description",
            verbose=True,
            memory=self.memory,
            return_intermediate_steps=True,
            agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
                          'suffix': SUFFIX}, )
        return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear)

    def run_text(self, text, state):
        self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500)
        res = self.agent({"input": text.strip()})
        res['output'] = res['output'].replace("\\", "/")
        response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output'])
        state = state + [(text, response)]
        print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
              f"Current Memory: {self.agent.memory.buffer}")
        return state, state

    def run_image(self, image, state, txt, lang):
        image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
        print("======>Auto Resize Image...")
        img = Image.open(image.name)
        width, height = img.size
        ratio = min(512 / width, 512 / height)
        width_new, height_new = (round(width * ratio), round(height * ratio))
        width_new = int(np.round(width_new / 64.0)) * 64
        height_new = int(np.round(height_new / 64.0)) * 64
        img = img.resize((width_new, height_new))
        img = img.convert('RGB')
        img.save(image_filename, "PNG")
        print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
        description = self.models['ImageCaptioning'].inference(image_filename)
        if lang == 'Chinese':
            Human_prompt = f'\nHuman: 提供一张名为 {image_filename}的图片。它的描述是: {description}。 这些信息帮助你理解这个图像，但是你应该使用工具来完成下面的任务，而不是直接从我的描述中想象。 如果你明白了, 说 \"收到\". \n'
            AI_prompt = "收到。  "
        else:
            Human_prompt = f'\nHuman: provide a figure named {image_filename}. The description is: {description}. This information helps you to understand this image, but you should use tools to finish following tasks, rather than directly imagine from my description. If you understand, say \"Received\". \n'
            AI_prompt = "Received.  "
        self.agent.memory.buffer = self.agent.memory.buffer + Human_prompt + 'AI: ' + AI_prompt
        state = state + [(f"![](file={image_filename})*{image_filename}*", AI_prompt)]
        print(f"\nProcessed run_image, Input image: {image_filename}\nCurrent state: {state}\n"
              f"Current Memory: {self.agent.memory.buffer}")
        return state, state, f'{txt} {image_filename} '

if __name__ == '__main__':
    if not os.path.exists("checkpoints"):
        os.mkdir("checkpoints")
    parser = argparse.ArgumentParser()
    parser.add_argument('--load', type=str, default="ImageCaptioning_cuda:0,Text2Image_cuda:0")
    args = parser.parse_args()
    load_dict = {e.split('_')[0].strip(): e.split('_')[1].strip() for e in args.load.split(',')}
    bot = ConversationBot(load_dict=load_dict)
    with gr.Blocks(css="#chatbot .overflow-y-auto{height:500px}") as demo:
        lang = gr.Radio(choices = ['Chinese','English'], value=None, label='Language')
        chatbot = gr.Chatbot(elem_id="chatbot", label="Visual ChatGPT")
        state = gr.State([])
        with gr.Row(visible=False) as input_raws:
            with gr.Column(scale=0.7):
                txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter, or upload an image").style(
                    container=False)
            with gr.Column(scale=0.15, min_width=0):
                clear = gr.Button("Clear")


LowCodeLLM.src.executingLLM

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from openAIWrapper import OpenAIWrapper

EXECUTING_LLM_PREFIX = """Executing LLM is designed to provide outstanding responses.
Executing LLM will be given a overall task as the background of the conversation between the Executing LLM and human.
When providing response, Executing LLM MUST STICTLY follow the provided standard operating procedure (SOP).
the SOP is formatted as:
'''
STEP 1: [step name][step descriptions][[[if 'condition1'][Jump to STEP]], [[if 'condition2'][Jump to STEP]], ...]
'''
here "[[[if 'condition1'][Jump to STEP n]]]" is judgmental logic. It means when you're performing this step, and if 'condition1' is satisfied, you will perform STEP n next.

Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

EXECUTING_LLM_SUFFIX = """
Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

class executingLLM:
    def __init__(self, temperature) -> None:
        self.prefix = EXECUTING_LLM_PREFIX
        self.suffix = EXECUTING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."},
                         {"role": "system", "content": self.prefix}]

    def execute(self, current_prompt, history):
        ''' provide LLM the dialogue history and the current prompt to get response '''
        messages = self.messages + history
        messages.append({'role': 'user', "content": current_prompt + self.suffix})
        response, status = self.LLM.run(messages)
        if status:
            return response
        else:
            return "OpenAI API error."


executingLLM

EXECUTING_LLM_PREFIX = """Executing LLM is designed to provide outstanding responses.
Executing LLM will be given a overall task as the background of the conversation between the Executing LLM and human.
When providing response, Executing LLM MUST STICTLY follow the provided standard operating procedure (SOP).
the SOP is formatted as:
'''
STEP 1: [step name][step descriptions][[[if 'condition1'][Jump to STEP]], [[if 'condition2'][Jump to STEP]], ...]
'''
here "[[[if 'condition1'][Jump to STEP n]]]" is judgmental logic. It means when you're performing this step, and if 'condition1' is satisfied, you will perform STEP n next.

Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

EXECUTING_LLM_SUFFIX = """
Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

class executingLLM:
    def __init__(self, temperature) -> None:
        self.prefix = EXECUTING_LLM_PREFIX
        self.suffix = EXECUTING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."},
                         {"role": "system", "content": self.prefix}]

    def execute(self, current_prompt, history):
        ''' provide LLM the dialogue history and the current prompt to get response '''
        messages = self.messages + history
        messages.append({'role': 'user', "content": current_prompt + self.suffix})
        response, status = self.LLM.run(messages)
        if status:
            return response
        else:
            return "OpenAI API error."


Executing LLM will be given a overall task as the background of the conversation between the Executing LLM and human.
When providing response, Executing LLM MUST STICTLY follow the provided standard operating procedure (SOP).
the SOP is formatted as:
'''
STEP 1: [step name][step descriptions][[[if 'condition1'][Jump to STEP]], [[if 'condition2'][Jump to STEP]], ...]
'''
here "[[[if 'condition1'][Jump to STEP n]]]" is judgmental logic. It means when you're performing this step, and if 'condition1' is satisfied, you will perform STEP n next.

Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

EXECUTING_LLM_SUFFIX = """
Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

class executingLLM:
    def __init__(self, temperature) -> None:
        self.prefix = EXECUTING_LLM_PREFIX
        self.suffix = EXECUTING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."},
                         {"role": "system", "content": self.prefix}]

    def execute(self, current_prompt, history):
        ''' provide LLM the dialogue history and the current prompt to get response '''
        messages = self.messages + history
        messages.append({'role': 'user', "content": current_prompt + self.suffix})
        response, status = self.LLM.run(messages)
        if status:
            return response
        else:
            return "OpenAI API error."


execute

Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

EXECUTING_LLM_SUFFIX = """
Remember:
Executing LLM is facing a real human, who does not know what SOP is.
So, Do not show him/her the SOP steps you are following, or it will make him/her confused. Just response the answer.
"""

class executingLLM:
    def __init__(self, temperature) -> None:
        self.prefix = EXECUTING_LLM_PREFIX
        self.suffix = EXECUTING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."},
                         {"role": "system", "content": self.prefix}]

    def execute(self, current_prompt, history):
        ''' provide LLM the dialogue history and the current prompt to get response '''
        messages = self.messages + history
        messages.append({'role': 'user', "content": current_prompt + self.suffix})
        response, status = self.LLM.run(messages)
        if status:
            return response
        else:
            return "OpenAI API error."


LowCodeLLM.src.app

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import os
from flask import Flask, request, send_from_directory
from flask_cors import CORS, cross_origin
from lowCodeLLM import lowCodeLLM
from flask.logging import default_handler
import logging

app = Flask('lowcode-llm', static_folder='', template_folder='')
app.debug = True
llm = lowCodeLLM()
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger = gunicorn_logger
logging_format = logging.Formatter(
    '%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s')
default_handler.setFormatter(logging_format)

@app.route("/")
def index():
    return send_from_directory(".", "index.html")

@app.route('/api/get_workflow', methods=['POST'])
@cross_origin()
def get_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        workflow = llm.get_workflow(task_prompt)
        return workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to get_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/extend_workflow', methods=['POST'])
@cross_origin()
def extend_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        current_workflow = request_content['current_workflow']
        step = request_content['step']
        sub_workflow = llm.extend_workflow(task_prompt, current_workflow, step)
        return sub_workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to extend_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/execute', methods=['POST'])
@cross_origin()
def execute():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        confirmed_workflow = request_content['confirmed_workflow']
        curr_input = request_content['curr_input']
        history = request_content['history']
        response = llm.execute(task_prompt,confirmed_workflow, history, curr_input)
        return response, 200
    except Exception as e:
        app.logger.error(
            'failed to execute, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500


index

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import os
from flask import Flask, request, send_from_directory
from flask_cors import CORS, cross_origin
from lowCodeLLM import lowCodeLLM
from flask.logging import default_handler
import logging

app = Flask('lowcode-llm', static_folder='', template_folder='')
app.debug = True
llm = lowCodeLLM()
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger = gunicorn_logger
logging_format = logging.Formatter(
    '%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s')
default_handler.setFormatter(logging_format)

@app.route("/")
def index():
    return send_from_directory(".", "index.html")

@app.route('/api/get_workflow', methods=['POST'])
@cross_origin()
def get_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        workflow = llm.get_workflow(task_prompt)
        return workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to get_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/extend_workflow', methods=['POST'])
@cross_origin()
def extend_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']


from flask_cors import CORS, cross_origin
from lowCodeLLM import lowCodeLLM
from flask.logging import default_handler
import logging

app = Flask('lowcode-llm', static_folder='', template_folder='')
app.debug = True
llm = lowCodeLLM()
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger = gunicorn_logger
logging_format = logging.Formatter(
    '%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s')
default_handler.setFormatter(logging_format)

@app.route("/")
def index():
    return send_from_directory(".", "index.html")

@app.route('/api/get_workflow', methods=['POST'])
@cross_origin()
def get_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        workflow = llm.get_workflow(task_prompt)
        return workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to get_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/extend_workflow', methods=['POST'])
@cross_origin()
def extend_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        current_workflow = request_content['current_workflow']
        step = request_content['step']
        sub_workflow = llm.extend_workflow(task_prompt, current_workflow, step)
        return sub_workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to extend_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/execute', methods=['POST'])
@cross_origin()
def execute():
    try:


@app.route("/")
def index():
    return send_from_directory(".", "index.html")

@app.route('/api/get_workflow', methods=['POST'])
@cross_origin()
def get_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        workflow = llm.get_workflow(task_prompt)
        return workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to get_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/extend_workflow', methods=['POST'])
@cross_origin()
def extend_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        current_workflow = request_content['current_workflow']
        step = request_content['step']
        sub_workflow = llm.extend_workflow(task_prompt, current_workflow, step)
        return sub_workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to extend_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/execute', methods=['POST'])
@cross_origin()
def execute():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        confirmed_workflow = request_content['confirmed_workflow']
        curr_input = request_content['curr_input']
        history = request_content['history']
        response = llm.execute(task_prompt,confirmed_workflow, history, curr_input)
        return response, 200
    except Exception as e:
        app.logger.error(
            'failed to execute, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500


execute

            'failed to get_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/extend_workflow', methods=['POST'])
@cross_origin()
def extend_workflow():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        current_workflow = request_content['current_workflow']
        step = request_content['step']
        sub_workflow = llm.extend_workflow(task_prompt, current_workflow, step)
        return sub_workflow, 200
    except Exception as e:
        app.logger.error(
            'failed to extend_workflow, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500

@app.route('/api/execute', methods=['POST'])
@cross_origin()
def execute():
    try:
        request_content = request.get_json()
        task_prompt = request_content['task_prompt']
        confirmed_workflow = request_content['confirmed_workflow']
        curr_input = request_content['curr_input']
        history = request_content['history']
        response = llm.execute(task_prompt,confirmed_workflow, history, curr_input)
        return response, 200
    except Exception as e:
        app.logger.error(
            'failed to execute, msg:%s, request data:%s' % (str(e), request.json))
        return {'errmsg': 'internal errors'}, 500


LowCodeLLM.src.openAIWrapper

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import os
import openai

class OpenAIWrapper:
    def __init__(self, temperature):
        self.key = os.environ.get("OPENAIKEY")
        openai.api_key = self.key

        # Access the USE_AZURE environment variable
        self.use_azure = os.environ.get('USE_AZURE')

        # Check if USE_AZURE is defined
        if self.use_azure is not None:
            # Convert the USE_AZURE value to boolean
            self.use_azure = self.use_azure.lower() == 'true'
        else:
            self.use_azure = False

        if self.use_azure:
            openai.api_type = "azure"
            self.api_base = os.environ.get('API_BASE')
            openai.api_base = self.api_base
            self.api_version = os.environ.get('API_VERSION')
            openai.api_version = self.api_version
            self.engine = os.environ.get('MODEL')
        else:
            self.chat_model_id = "gpt-3.5-turbo"

        self.temperature = temperature
        self.max_tokens = 2048
        self.top_p = 1
        self.time_out = 7

    def run(self, prompt):
        return self._post_request_chat(prompt)

    def _post_request_chat(self, messages):
        try:
            if self.use_azure:
                response = openai.ChatCompletion.create(
                    engine=self.engine,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            else:
                response = openai.ChatCompletion.create(
                    model=self.chat_model_id,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            res = response['choices'][0]['message']['content']
            return res, True
        except Exception as e:
            return "", False


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import os
import openai

class OpenAIWrapper:
    def __init__(self, temperature):
        self.key = os.environ.get("OPENAIKEY")
        openai.api_key = self.key

        # Access the USE_AZURE environment variable
        self.use_azure = os.environ.get('USE_AZURE')

        # Check if USE_AZURE is defined
        if self.use_azure is not None:
            # Convert the USE_AZURE value to boolean
            self.use_azure = self.use_azure.lower() == 'true'
        else:
            self.use_azure = False

        if self.use_azure:
            openai.api_type = "azure"
            self.api_base = os.environ.get('API_BASE')
            openai.api_base = self.api_base
            self.api_version = os.environ.get('API_VERSION')
            openai.api_version = self.api_version
            self.engine = os.environ.get('MODEL')
        else:
            self.chat_model_id = "gpt-3.5-turbo"

        self.temperature = temperature
        self.max_tokens = 2048
        self.top_p = 1
        self.time_out = 7

    def run(self, prompt):
        return self._post_request_chat(prompt)

    def _post_request_chat(self, messages):
        try:
            if self.use_azure:
                response = openai.ChatCompletion.create(
                    engine=self.engine,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            else:
                response = openai.ChatCompletion.create(
                    model=self.chat_model_id,
                    messages=messages,
                    temperature=self.temperature,


run

            # Convert the USE_AZURE value to boolean
            self.use_azure = self.use_azure.lower() == 'true'
        else:
            self.use_azure = False

        if self.use_azure:
            openai.api_type = "azure"
            self.api_base = os.environ.get('API_BASE')
            openai.api_base = self.api_base
            self.api_version = os.environ.get('API_VERSION')
            openai.api_version = self.api_version
            self.engine = os.environ.get('MODEL')
        else:
            self.chat_model_id = "gpt-3.5-turbo"

        self.temperature = temperature
        self.max_tokens = 2048
        self.top_p = 1
        self.time_out = 7

    def run(self, prompt):
        return self._post_request_chat(prompt)

    def _post_request_chat(self, messages):
        try:
            if self.use_azure:
                response = openai.ChatCompletion.create(
                    engine=self.engine,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            else:
                response = openai.ChatCompletion.create(
                    model=self.chat_model_id,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0


            self.use_azure = False

        if self.use_azure:
            openai.api_type = "azure"
            self.api_base = os.environ.get('API_BASE')
            openai.api_base = self.api_base
            self.api_version = os.environ.get('API_VERSION')
            openai.api_version = self.api_version
            self.engine = os.environ.get('MODEL')
        else:
            self.chat_model_id = "gpt-3.5-turbo"

        self.temperature = temperature
        self.max_tokens = 2048
        self.top_p = 1
        self.time_out = 7

    def run(self, prompt):
        return self._post_request_chat(prompt)

    def _post_request_chat(self, messages):
        try:
            if self.use_azure:
                response = openai.ChatCompletion.create(
                    engine=self.engine,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            else:
                response = openai.ChatCompletion.create(
                    model=self.chat_model_id,
                    messages=messages,
                    temperature=self.temperature,
                    max_tokens=self.max_tokens,
                    frequency_penalty=0,
                    presence_penalty=0
                )
            res = response['choices'][0]['message']['content']
            return res, True
        except Exception as e:
            return "", False


LowCodeLLM.src.planningLLM

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import re
import json
from openAIWrapper import OpenAIWrapper

PLANNING_LLM_PREFIX = """Planning LLM is designed to provide a standard operating procedure so that an difficult task will be broken down into several steps, and the task will be easily solved by following these steps.
Planning LLM is a powerful problem-solving assistant, and it only needs to analyze the task and provide standard operating procedure as guidance, but does not need actually to solve the problem.
Sometimes there exists some unknown or undetermined situation, thus judgmental logic is needed: some "conditions" are listed, and the next step that should be carried out if a "condition" is satisfied is also listed. The judgmental logics are not necessary.
Planning LLM MUST only provide standard operating procedure in the following format without any other words:
'''
STEP 1: [step name][step descriptions][[[if 'condition1'][Jump to STEP]], [[[if 'condition1'][Jump to STEP]], [[if 'condition2'][Jump to STEP]], ...]
STEP 2: [step name][step descriptions][[[if 'condition1'][Jump to STEP]], [[[if 'condition1'][Jump to STEP]], [[if 'condition2'][Jump to STEP]], ...]
...
'''

For example:
'''
STEP 1: [Brainstorming][Choose a topic or prompt, and generate ideas and organize them into an outline][]
STEP 2: [Research][Gather information, take notes and organize them into the outline][[[lack of ideas][Jump to STEP 1]]]
...
'''
"""

EXTEND_PREFIX = """
\nsome steps of the SOP provided by Planning LLM are too rough, so Planning LLM can also provide a detailed sub-SOP for the given step.
Remember, Planning LLM take the overall SOP into consideration, and the sub-SOP MUST be consistent with the rest of the steps, and there MUST be no duplication in content between the extension and the original SOP.

For example:
If the overall SOP is:
'''
STEP 1: [Brainstorming][Choose a topic or prompt, and generate ideas and organize them into an outline][]
STEP 2: [Write][write the text][]
'''
If the STEP 2: "write the text" is too rough and needs to be extended, then the response could be:
'''
STEP 2.1: [Write the title][write the title of the essay][]
STEP 2.2: [Write the body][write the body of the essay][[[if lack of materials][Jump to STEP 1]]]
'''
"""

PLANNING_LLM_SUFFIX = """\nRemember: Planning LLM is very strict to the format and NEVER reply any word other than the standard operating procedure.
"""

class planningLLM:
    def __init__(self, temperature) -> None:
        self.prefix = PLANNING_LLM_PREFIX
        self.suffix = PLANNING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."}]

    def get_workflow(self, task_prompt):
        '''
        - input: task_prompt
        - output: workflow (json)
        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def _txt2json(self, workflow_txt):
        ''' convert the workflow in natural language to json format '''
        workflow = []
        try:
            steps = workflow_txt.split('\n')
            for step in steps:
                if step[0:4] != "STEP":
                    continue
                left_indices = [_.start() for _ in re.finditer("\[", step)]
                right_indices = [_.start() for _ in re.finditer("\]", step)]
                step_id = step[: left_indices[0]-2]
                step_name = step[left_indices[0]+1: right_indices[0]]
                step_description = step[left_indices[1]+1: right_indices[1]]
                jump_str = step[left_indices[2]+1: right_indices[-1]]
                if re.findall(re.compile(r'[A-Za-z]',re.S), jump_str) == []:
                    workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": [], "extension": []})
                    continue
                jump_logic = []
                left_indices = [_.start() for _ in re.finditer('\[', jump_str)]
                right_indices = [_.start() for _ in re.finditer('\]', jump_str)]
                i = 1
                while i < len(left_indices):
                    jump = {"Condition": jump_str[left_indices[i]+1: right_indices[i-1]], "Target": re.search(r'STEP\s\d', jump_str[left_indices[i+1]+1: right_indices[i]]).group(0)}
                    jump_logic.append(jump)
                    i += 3
                workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": jump_logic, "extension": []})
            return json.dumps(workflow)
        except:
            print("Format error, please try again.")


planningLLM

EXTEND_PREFIX = """
\nsome steps of the SOP provided by Planning LLM are too rough, so Planning LLM can also provide a detailed sub-SOP for the given step.
Remember, Planning LLM take the overall SOP into consideration, and the sub-SOP MUST be consistent with the rest of the steps, and there MUST be no duplication in content between the extension and the original SOP.

For example:
If the overall SOP is:
'''
STEP 1: [Brainstorming][Choose a topic or prompt, and generate ideas and organize them into an outline][]
STEP 2: [Write][write the text][]
'''
If the STEP 2: "write the text" is too rough and needs to be extended, then the response could be:
'''
STEP 2.1: [Write the title][write the title of the essay][]
STEP 2.2: [Write the body][write the body of the essay][[[if lack of materials][Jump to STEP 1]]]
'''
"""

PLANNING_LLM_SUFFIX = """\nRemember: Planning LLM is very strict to the format and NEVER reply any word other than the standard operating procedure.
"""

class planningLLM:
    def __init__(self, temperature) -> None:
        self.prefix = PLANNING_LLM_PREFIX
        self.suffix = PLANNING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."}]

    def get_workflow(self, task_prompt):
        '''
        - input: task_prompt
        - output: workflow (json)
        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def _txt2json(self, workflow_txt):
        ''' convert the workflow in natural language to json format '''
        workflow = []
        try:
            steps = workflow_txt.split('\n')
            for step in steps:
                if step[0:4] != "STEP":
                    continue
                left_indices = [_.start() for _ in re.finditer("\[", step)]
                right_indices = [_.start() for _ in re.finditer("\]", step)]
                step_id = step[: left_indices[0]-2]
                step_name = step[left_indices[0]+1: right_indices[0]]
                step_description = step[left_indices[1]+1: right_indices[1]]
                jump_str = step[left_indices[2]+1: right_indices[-1]]
                if re.findall(re.compile(r'[A-Za-z]',re.S), jump_str) == []:
                    workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": [], "extension": []})
                    continue
                jump_logic = []
                left_indices = [_.start() for _ in re.finditer('\[', jump_str)]
                right_indices = [_.start() for _ in re.finditer('\]', jump_str)]
                i = 1
                while i < len(left_indices):
                    jump = {"Condition": jump_str[left_indices[i]+1: right_indices[i-1]], "Target": re.search(r'STEP\s\d', jump_str[left_indices[i+1]+1: right_indices[i]]).group(0)}
                    jump_logic.append(jump)
                    i += 3
                workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": jump_logic, "extension": []})
            return json.dumps(workflow)
        except:
            print("Format error, please try again.")


\nsome steps of the SOP provided by Planning LLM are too rough, so Planning LLM can also provide a detailed sub-SOP for the given step.
Remember, Planning LLM take the overall SOP into consideration, and the sub-SOP MUST be consistent with the rest of the steps, and there MUST be no duplication in content between the extension and the original SOP.

For example:
If the overall SOP is:
'''
STEP 1: [Brainstorming][Choose a topic or prompt, and generate ideas and organize them into an outline][]
STEP 2: [Write][write the text][]
'''
If the STEP 2: "write the text" is too rough and needs to be extended, then the response could be:
'''
STEP 2.1: [Write the title][write the title of the essay][]
STEP 2.2: [Write the body][write the body of the essay][[[if lack of materials][Jump to STEP 1]]]
'''
"""

PLANNING_LLM_SUFFIX = """\nRemember: Planning LLM is very strict to the format and NEVER reply any word other than the standard operating procedure.
"""

class planningLLM:
    def __init__(self, temperature) -> None:
        self.prefix = PLANNING_LLM_PREFIX
        self.suffix = PLANNING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."}]

    def get_workflow(self, task_prompt):
        '''
        - input: task_prompt
        - output: workflow (json)
        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)


STEP 1: [Brainstorming][Choose a topic or prompt, and generate ideas and organize them into an outline][]
STEP 2: [Write][write the text][]
'''
If the STEP 2: "write the text" is too rough and needs to be extended, then the response could be:
'''
STEP 2.1: [Write the title][write the title of the essay][]
STEP 2.2: [Write the body][write the body of the essay][[[if lack of materials][Jump to STEP 1]]]
'''
"""

PLANNING_LLM_SUFFIX = """\nRemember: Planning LLM is very strict to the format and NEVER reply any word other than the standard operating procedure.
"""

class planningLLM:
    def __init__(self, temperature) -> None:
        self.prefix = PLANNING_LLM_PREFIX
        self.suffix = PLANNING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."}]

    def get_workflow(self, task_prompt):
        '''
        - input: task_prompt
        - output: workflow (json)
        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def _txt2json(self, workflow_txt):
        ''' convert the workflow in natural language to json format '''
        workflow = []
        try:
            steps = workflow_txt.split('\n')
            for step in steps:
                if step[0:4] != "STEP":


class planningLLM:
    def __init__(self, temperature) -> None:
        self.prefix = PLANNING_LLM_PREFIX
        self.suffix = PLANNING_LLM_SUFFIX
        self.LLM = OpenAIWrapper(temperature)
        self.messages = [{"role": "system", "content": "You are a helpful assistant."}]

    def get_workflow(self, task_prompt):
        '''
        - input: task_prompt
        - output: workflow (json)
        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def _txt2json(self, workflow_txt):
        ''' convert the workflow in natural language to json format '''
        workflow = []
        try:
            steps = workflow_txt.split('\n')
            for step in steps:
                if step[0:4] != "STEP":
                    continue
                left_indices = [_.start() for _ in re.finditer("\[", step)]
                right_indices = [_.start() for _ in re.finditer("\]", step)]
                step_id = step[: left_indices[0]-2]
                step_name = step[left_indices[0]+1: right_indices[0]]
                step_description = step[left_indices[1]+1: right_indices[1]]
                jump_str = step[left_indices[2]+1: right_indices[-1]]
                if re.findall(re.compile(r'[A-Za-z]',re.S), jump_str) == []:
                    workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": [], "extension": []})
                    continue
                jump_logic = []
                left_indices = [_.start() for _ in re.finditer('\[', jump_str)]


        '''
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def extend_workflow(self, task_prompt, current_workflow, step):
        messages = self.messages + [{'role': 'user', "content": PLANNING_LLM_PREFIX+'\nThe task is:\n'+task_prompt+PLANNING_LLM_SUFFIX}]
        messages.append({'role': 'user', "content": EXTEND_PREFIX+
                         'The current SOP is:\n'+current_workflow+
                         '\nThe step needs to be extended is:\n'+step+
                         PLANNING_LLM_SUFFIX})
        response, status = self.LLM.run(messages)
        if status:
            return self._txt2json(response)
        else:
            return "OpenAI API error."

    def _txt2json(self, workflow_txt):
        ''' convert the workflow in natural language to json format '''
        workflow = []
        try:
            steps = workflow_txt.split('\n')
            for step in steps:
                if step[0:4] != "STEP":
                    continue
                left_indices = [_.start() for _ in re.finditer("\[", step)]
                right_indices = [_.start() for _ in re.finditer("\]", step)]
                step_id = step[: left_indices[0]-2]
                step_name = step[left_indices[0]+1: right_indices[0]]
                step_description = step[left_indices[1]+1: right_indices[1]]
                jump_str = step[left_indices[2]+1: right_indices[-1]]
                if re.findall(re.compile(r'[A-Za-z]',re.S), jump_str) == []:
                    workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": [], "extension": []})
                    continue
                jump_logic = []
                left_indices = [_.start() for _ in re.finditer('\[', jump_str)]
                right_indices = [_.start() for _ in re.finditer('\]', jump_str)]
                i = 1
                while i < len(left_indices):
                    jump = {"Condition": jump_str[left_indices[i]+1: right_indices[i-1]], "Target": re.search(r'STEP\s\d', jump_str[left_indices[i+1]+1: right_indices[i]]).group(0)}
                    jump_logic.append(jump)
                    i += 3
                workflow.append({"stepId": step_id, "stepName": step_name, "stepDescription": step_description, "jumpLogic": jump_logic, "extension": []})
            return json.dumps(workflow)
        except:
            print("Format error, please try again.")


LowCodeLLM.src.lowCodeLLM

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from planningLLM import planningLLM
from executingLLM import executingLLM
import json

class lowCodeLLM:
    def __init__(self, PLLM_temperature=0.4, ELLM_temperature=0):
        self.PLLM = planningLLM(PLLM_temperature)
        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""
            step_res += step["stepId"] + ": [" + step["stepName"] + "]"
            step_res += "[" + step["stepDescription"] + "]["
            for jump in step["jumpLogic"]:
                step_res += "[[" + jump["Condition"] + "][" + jump["Target"] + "]],"
            step_res += "]\n"
            return step_res

        workflow_txt = ""
        for step in json.loads(workflow_json):
            workflow_txt += json2text_step(step)
            for substep in step['extension']:
                workflow_txt += json2text_step(substep)
        return workflow_txt


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from planningLLM import planningLLM
from executingLLM import executingLLM
import json

class lowCodeLLM:
    def __init__(self, PLLM_temperature=0.4, ELLM_temperature=0):
        self.PLLM = planningLLM(PLLM_temperature)
        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from planningLLM import planningLLM
from executingLLM import executingLLM
import json

class lowCodeLLM:
    def __init__(self, PLLM_temperature=0.4, ELLM_temperature=0):
        self.PLLM = planningLLM(PLLM_temperature)
        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from planningLLM import planningLLM
from executingLLM import executingLLM
import json

class lowCodeLLM:
    def __init__(self, PLLM_temperature=0.4, ELLM_temperature=0):
        self.PLLM = planningLLM(PLLM_temperature)
        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""
            step_res += step["stepId"] + ": [" + step["stepName"] + "]"
            step_res += "[" + step["stepDescription"] + "]["
            for jump in step["jumpLogic"]:
                step_res += "[[" + jump["Condition"] + "][" + jump["Target"] + "]],"
            step_res += "]\n"
            return step_res


execute

from planningLLM import planningLLM
from executingLLM import executingLLM
import json

class lowCodeLLM:
    def __init__(self, PLLM_temperature=0.4, ELLM_temperature=0):
        self.PLLM = planningLLM(PLLM_temperature)
        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""
            step_res += step["stepId"] + ": [" + step["stepName"] + "]"
            step_res += "[" + step["stepDescription"] + "]["
            for jump in step["jumpLogic"]:
                step_res += "[[" + jump["Condition"] + "][" + jump["Target"] + "]],"
            step_res += "]\n"
            return step_res

        workflow_txt = ""
        for step in json.loads(workflow_json):
            workflow_txt += json2text_step(step)
            for substep in step['extension']:
                workflow_txt += json2text_step(substep)
        return workflow_txt


        self.ELLM = executingLLM(ELLM_temperature)

    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""
            step_res += step["stepId"] + ": [" + step["stepName"] + "]"
            step_res += "[" + step["stepDescription"] + "]["
            for jump in step["jumpLogic"]:
                step_res += "[[" + jump["Condition"] + "][" + jump["Target"] + "]],"
            step_res += "]\n"
            return step_res

        workflow_txt = ""
        for step in json.loads(workflow_json):
            workflow_txt += json2text_step(step)
            for substep in step['extension']:
                workflow_txt += json2text_step(substep)
        return workflow_txt


    def get_workflow(self, task_prompt):
        return self.PLLM.get_workflow(task_prompt)

    def extend_workflow(self, task_prompt, current_workflow, step=''):
        ''' generate a sub-workflow for one of steps
            - input: the current workflow, the step needs to extend
            - output: sub-workflow '''
        workflow = self._json2txt(current_workflow)
        return self.PLLM.extend_workflow(task_prompt, workflow, step)

    def execute(self, task_prompt,confirmed_workflow, history, curr_input):
        ''' chat with the workflow-equipped low-code LLM '''
        prompt = [{'role': 'system', "content": 'The overall task you are facing is: '+task_prompt+
                '\nThe standard operating procedure(SOP) is:\n'+self._json2txt(confirmed_workflow)}]
        history = prompt + history
        response = self.ELLM.execute(curr_input, history)
        return response

    def _json2txt(self, workflow_json):
        ''' convert the json workflow to text'''
        def json2text_step(step):
            step_res = ""
            step_res += step["stepId"] + ": [" + step["stepName"] + "]"
            step_res += "[" + step["stepDescription"] + "]["
            for jump in step["jumpLogic"]:
                step_res += "[[" + jump["Condition"] + "][" + jump["Target"] + "]],"
            step_res += "]\n"
            return step_res

        workflow_txt = ""
        for step in json.loads(workflow_json):
            workflow_txt += json2text_step(step)
            for substep in step['extension']:
                workflow_txt += json2text_step(substep)
        return workflow_txt


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import json
import sys
import os
sys.path.append(os.getcwd())

def test_get_workflow():
    from lowCodeLLM import lowCodeLLM
    cases = json.load(open("./test/testcases/get_workflow_test_cases.json", "r"))
    llm = lowCodeLLM(0.5, 0)
    for c in cases:
        task_prompt = c["task_prompt"]
        result = llm.get_workflow(task_prompt)
        assert len(json.loads(result)) >= 1


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import json
import sys
import os
import time
sys.path.append(os.getcwd())

def test_extend_workflow():
    from lowCodeLLM import lowCodeLLM
    cases = json.load(open("./test/testcases/execute_test_cases.json", "r"))
    llm = lowCodeLLM(0.5, 0)
    for c in cases:
        task_prompt = c["task_prompt"]
        confirmed_workflow = c["confirmed_workflow"]
        history = c["history"]
        curr_input = c["curr_input"]
        result = llm.execute(task_prompt, confirmed_workflow, history, curr_input)
        time.sleep(5)
        assert type(result) == str
        assert len(result) > 0