Download app.py from Timilehin/DME: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/Timilehin/DME/resolve/main/app.py
- Command line
-
hf download hf://spaces/Timilehin/DME/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Timilehin/DME/resolve/main/app.py
11.6 kB
| from fastapi import FastAPI | |
| import gradio as gr | |
| from PIL import Image | |
| from ultralytics import YOLO | |
| import easyocr | |
| import cv2 | |
| from warnings import filterwarnings | |
| import numpy as np | |
| import re | |
| import json | |
| import glob | |
| import os | |
| import shutil | |
| import paddleocr | |
| from paddleocr import PaddleOCR | |
| from rich import print | |
| from langchain.chains import ConversationChain | |
| from langchain.memory import ConversationBufferMemory | |
| from langchain_core.messages import HumanMessage | |
| from langchain_groq import ChatGroq # Import ChatGroq | |
| from langchain_openai import ChatOpenAI | |
| import base64 | |
| from fastapi.middleware.cors import CORSMiddleware | |
| from warnings import filterwarnings | |
| filterwarnings('ignore') | |
| # Initialize the memory | |
| memory = ConversationBufferMemory() | |
| model_key = os.getenv("API_KEY") | |
| model_id = "gpt-4o-mini" | |
| # Initialize the ChatGroq model | |
| # chat_model = ChatGroq(model="llama-3.2-11b-vision-preview", temperature=0, | |
| # api_key="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", max_tokens=1000) | |
| chat_model = ChatOpenAI(model=model_id, max_tokens=1000, openai_api_key=model_key) | |
| filterwarnings('ignore') | |
| # Create a reader to do OCR. | |
| reader = easyocr.Reader(['en']) | |
| # Load a pretrained YOLOv8n model | |
| model = YOLO('yolo_models/yolov8l_imgsz800_map85_best.pt', verbose=False) | |
| print('Model Loaded...') | |
| ocr_model = PaddleOCR(use_angle_cls=True, lang='en', show_log=False, | |
| rec_model_dir='rec_r45_abinet', det_model_dir='det_r50_vd_east_v2.0') | |
| def perform_pad_ocr(image, conf=0.20, iou=0.91, imgsz=800): | |
| if os.path.exists('cropped'): | |
| shutil.rmtree('cropped/') | |
| # make detection | |
| results = model.predict(image, conf=conf, iou=iou, imgsz=imgsz) | |
| ocr_result_path = "OCR_RESULTS" | |
| image_fname = "uploaded_image" | |
| chat1 = [] | |
| timestamp_formats = ['%I.%M %p', '%I %p', '%I.%M%p', '%I%M %p', '%I %p', '%I %M %p', | |
| '%I:%M %p', '%I:%M:%S %p', '%H:%M', '%H.%M', '%H %M', '%H:%M:%S', '%H.%M.%S', | |
| '%H %M %S', '%H:%M:%S.%f', '%H.%M.%S.%f', '%H %M %S %f'] | |
| for j in results: | |
| os.makedirs(ocr_result_path, exist_ok=True) | |
| boxes = j.boxes.cpu().numpy() | |
| j.save(filename=f'{ocr_result_path}/{image_fname}_result.jpg') # save to disk | |
| j.save_crop('cropped') | |
| message_boxes = [box for box in boxes if j.names[int(box.cls[0])] in ['sender_message', 'receiver_message']] | |
| replied_boxes = [box for box in boxes if j.names[int(box.cls[0])] in ['replied_to_message']] | |
| message_boxes.sort(key=lambda box: box.xyxy[0][1]) | |
| replied_boxes.sort(key=lambda box: box.xyxy[0][1]) | |
| emoji_boxes = [box for box in boxes if j.names[int(box.cls[0])] == 'emoji'] | |
| timestamp_boxes = [box for box in boxes if j.names[int(box.cls[0])] == 'timestamp'] | |
| for msg_box, t_box in zip(message_boxes, timestamp_boxes): | |
| message_type = j.names[int(msg_box.cls[0])] | |
| msg_r = msg_box.xyxy[0].astype(int) | |
| roi = (cv2.imread(image, 0))[msg_r[1]:msg_r[3], msg_r[0]:msg_r[2]] | |
| ocr_res = ocr_model.ocr(roi, cls=True)[0] | |
| ocr_text3 = " " | |
| if ocr_res is not None: | |
| for i in ocr_res: | |
| txt = i[-1][0] | |
| txt = txt.replace('\n', ' ') | |
| txt = txt.strip() | |
| ocr_text3 += f"{txt} " | |
| t_i = cv2.imread(image) | |
| fin_oc = [] | |
| ocr_text1 = " " | |
| for i,emoji_box in enumerate(emoji_boxes): | |
| emoji_r = emoji_box.xyxy[0].astype(int) | |
| kernel = np.ones((8,8),np.float32)/15 | |
| t_i[emoji_r[1]:emoji_r[3], emoji_r[0]:emoji_r[2]] = cv2.filter2D(t_i[emoji_r[1]:emoji_r[3], emoji_r[0]:emoji_r[2]], -1, kernel) | |
| center_x = int((emoji_r[0] + emoji_r[2]) / 2) | |
| center_y = int((emoji_r[1] + emoji_r[3]) / 2) | |
| text_size = cv2.getTextSize("%", cv2.FONT_HERSHEY_SIMPLEX, 1, 2)[0] | |
| text_x = center_x - text_size[0] // 2 | |
| text_y = center_y + text_size[1] // 2 | |
| cv2.putText(t_i, "% ", (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0) , 2) | |
| cv2.imwrite('t_i.jpg', t_i) | |
| roi1 = (t_i[msg_r[1]:msg_r[3], msg_r[0]:msg_r[2]]) | |
| ocr_res1 = ocr_model.ocr(roi1, cls=True)[0] | |
| new_oc = [] | |
| if ocr_res1 is not None: | |
| for i in ocr_res1: | |
| txt1 = i[-1][0] | |
| txt1 = txt1.replace('\n', ' ') | |
| txt1 = txt1.strip() | |
| ocr_text1 += f"{txt1} " | |
| new_oc.append(txt1) | |
| fin_oc.append(new_oc) | |
| ocr_text2 = " ".join(fin_oc[-1]) | |
| ocr_text3 = ocr_text2 | |
| ocr_text3 = re.sub(r'(EA|EF|EL|EM|EI|EN|EV|El|FM|FN|FI|FA|FE|FV|M E|E M|EE|MM|NM|MN|MI|NN|IM|M1|1M|MW|AN|SN|Q|%)+', '$$ ', ocr_text3) | |
| rep_tt = "" | |
| for replied_box in replied_boxes: | |
| rep_r = replied_box.xyxy[0].astype(int) | |
| if (msg_r[0] <= rep_r[0] and rep_r[2] <= msg_r[2]) and (msg_r[1] <= rep_r[1] and msg_r[3] <= rep_r[3]): | |
| rep_roi = cv2.imread(image, 0)[rep_r[1]:rep_r[3], rep_r[0]:rep_r[2]] | |
| rep_ocr_res = ocr_model.ocr(rep_roi, cls=True)[0] | |
| if rep_ocr_res is not None: | |
| for i in rep_ocr_res: | |
| txt = i[-1][0] | |
| rep_tt += f"{txt} " | |
| try: | |
| for time_box in timestamp_boxes: | |
| time_r = time_box.xyxy[0].astype(int) | |
| if (msg_r[0] <= time_r[0] and time_r[2] <= msg_r[2]) and (msg_r[1] <= time_r[1] and time_r[3] <= msg_r[3]): | |
| ts_r = time_r | |
| ts_roi = cv2.imread(image)[ts_r[1]:ts_r[3], ts_r[0]:ts_r[2]] | |
| ts_ocr_res = ocr_model.ocr(ts_roi, cls=True)[0] | |
| ts_text = "" | |
| if ts_ocr_res is not None: | |
| for i in ts_ocr_res: | |
| txt = i[-1][0] | |
| ts_text += txt | |
| timestamp = txt | |
| if timestamp is None: | |
| match = re.search(r'\b\d{1,2}:\d{2}\s?(?:AM|PM)\b', ocr_text3) | |
| timestamp = match.group() if match else None | |
| ocr_text3 = ocr_text3.replace(timestamp, ' ') if timestamp else ocr_text3 | |
| ocr_text3 = re.sub(r'\b\d{1,2}:\d{2}\s?(?:AM|PM)\b', '', ocr_text3) | |
| ocr_text3 = ocr_text1.replace(rep_tt, '') if rep_tt else ocr_text3 | |
| dictt1 = {"type": message_type, "text": ocr_text3.strip(), 'timestamp': timestamp} | |
| chat1.append(dictt1) | |
| except: | |
| pass | |
| json.dump(chat1, open(f"{ocr_result_path}/{image_fname}_result1.json", 'w')) | |
| return chat1 | |
| def get_deep_message_extractor_ans(image_path, q): | |
| # Construct the prompt with memory | |
| fin_q = json.dumps(q, indent=2) | |
| with open(image_path, "rb") as image_file: | |
| image_base64 = base64.b64encode(image_file.read()).decode('utf-8') | |
| prompt = ( | |
| f"From the original image" | |
| f"You are a deep message and emoji extractor assistant.\n" | |
| f"Your task is to process and analyze messages extracted text \n{fin_q} \n: from that image." | |
| f"Your main focus is just removing duplicate dictionary with the following condition: \n" | |
| f"If more than two dicts shares same text: values, remove the second dict and keep the first one\n" | |
| f"Ensure to remove duplicates in dictionary, before outputing final JSON\n" | |
| f"Do not modify the extracted texts; more importantly do not remove the emoji placeholders $$\n" | |
| f"Ensure your final output is a JSON just like you receive it.\n" | |
| f"If there are no duplicates, return original JSON with no modification\n" | |
| f"If there are duplicates, JUST REMOVE the later dictionary and return final JSON, *** DO NOT MODIFY THE TEXTS, EXPECIALLY THE emoji placeholders $$ ***\n" | |
| f"If you find emoji in the image, but not in the original extracted text, also have it replace with the placeholder $$\n" | |
| f"Finally; make sure all the $$ emoji placeholders are replaced with unicode emoji in the final text or JSON\n" | |
| f"And if no placeholder found, check from the image and make sure unicode emoji are correctly placed in the texts if available" | |
| ) | |
| # Add the conversation history from memory | |
| history = memory.load_memory_variables({}) | |
| if history['history']: | |
| prompt = f"{history['history']}\n\n{prompt}" | |
| # Invoke the ChatGroq model | |
| # response = chat_model.invoke([HumanMessage(content=prompt)]).content.strip() | |
| response = chat_model.invoke([ | |
| HumanMessage( | |
| content=[ | |
| {"type": "text", "text": prompt}, | |
| {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}}, | |
| ] | |
| ) | |
| ]) | |
| # Save the current interaction to memory | |
| memory.save_context({"input": q}, {"output": response.content.strip()}) | |
| return response.content.strip() | |
| def extract_json_from_response(response_text): | |
| """ | |
| Extracts the JSON portion from the LLM's response. | |
| """ | |
| # Use regex to find the JSON object or array | |
| json_match = re.search(r'\[\s*\{.*\}\s*\]|\{\s*.*\s*\}', response_text, re.DOTALL) | |
| if json_match: | |
| try: | |
| # Parse the extracted JSON | |
| json_data = json.loads(json_match.group()) | |
| return json_data | |
| except json.JSONDecodeError: | |
| print("Failed to parse extracted JSON.") | |
| return {"error": "Invalid JSON format in the response."} | |
| else: | |
| return {"error": "No JSON found in the response."} | |
| def gradio_interface(image, conf, iou, imgsz): | |
| res = perform_pad_ocr(image, conf, iou, imgsz) | |
| fin_res = get_deep_message_extractor_ans(image, res) | |
| ex_res = extract_json_from_response(fin_res) | |
| # print(res) | |
| return ex_res | |
| # Create the Gradio interface | |
| iface = gr.Interface( | |
| fn=gradio_interface, | |
| inputs=[ | |
| gr.Image(type="filepath", label="Upload Image"), | |
| gr.Slider(minimum=0.25, maximum=1.0, step=0.01, value=0.25, label="Confidence Threshold"), | |
| gr.Slider(minimum=0.45, maximum=1.0, step=0.01, value=0.91, label="IOU Threshold"), | |
| gr.Slider(minimum=800, maximum=1000, step=32, value=800, label="Image Size") | |
| ], | |
| outputs=gr.JSON(label="OCR Results"), | |
| title="deep message extractor", | |
| description="Upload an image and adjust the parameters to extract text." | |
| ) | |
| # # Create the FastAPI app | |
| # app = FastAPI() | |
| # # Add CORS middleware to allow all origins | |
| # app.add_middleware( | |
| # CORSMiddleware, | |
| # allow_origins=["*"], | |
| # allow_credentials=True, | |
| # allow_methods=["*"], | |
| # allow_headers=["*"], | |
| # ) | |
| # # Mount the Gradio app | |
| # app = gr.mount_gradio_app(app, iface, path="/") | |
| # # Run the FastAPI app | |
| # if __name__ == "__main__": | |
| # import uvicorn | |
| # uvicorn.run(app, host="0.0.0.0", port=80) | |
| iface.launch(favicon_path='smile.png') |