Timilehin commited on
Commit
e52dcae
·
verified ·
1 Parent(s): dcdb087

Upload 8 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ smile.png filter=lfs diff=lfs merge=lfs -text
37
+ test_images/ss_test1.png filter=lfs diff=lfs merge=lfs -text
38
+ test_images/ss_test2.png filter=lfs diff=lfs merge=lfs -text
39
+ test_images/ss_test3.png filter=lfs diff=lfs merge=lfs -text
40
+ test_images/ss_test4.png filter=lfs diff=lfs merge=lfs -text
app.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ import gradio as gr
3
+ from PIL import Image
4
+ from ultralytics import YOLO
5
+ import easyocr
6
+ import cv2
7
+ from warnings import filterwarnings
8
+ import numpy as np
9
+ import re
10
+ import json
11
+ import glob
12
+ import os
13
+ import shutil
14
+ import paddleocr
15
+ from paddleocr import PaddleOCR
16
+ from rich import print
17
+ from langchain.chains import ConversationChain
18
+ from langchain.memory import ConversationBufferMemory
19
+ from langchain_core.messages import HumanMessage
20
+ from langchain_groq import ChatGroq # Import ChatGroq
21
+ from langchain_openai import ChatOpenAI
22
+ import base64
23
+ from fastapi.middleware.cors import CORSMiddleware
24
+ from warnings import filterwarnings
25
+
26
+ filterwarnings('ignore')
27
+
28
+ # Initialize the memory
29
+ memory = ConversationBufferMemory()
30
+
31
+ model_key = "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"
32
+ model_id = "gpt-4o-mini"
33
+
34
+ # Initialize the ChatGroq model
35
+ # chat_model = ChatGroq(model="llama-3.2-11b-vision-preview", temperature=0,
36
+ # api_key="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", max_tokens=1000)
37
+ chat_model = ChatOpenAI(model=model_id, max_tokens=1000, openai_api_key=model_key)
38
+
39
+ filterwarnings('ignore')
40
+
41
+ # Create a reader to do OCR.
42
+ reader = easyocr.Reader(['en'])
43
+
44
+ # Load a pretrained YOLOv8n model
45
+ model = YOLO('yolo_models/yolov8l_imgsz800_map85_best.pt', verbose=False)
46
+ print('Model Loaded...')
47
+
48
+ ocr_model = PaddleOCR(use_angle_cls=True, lang='en', show_log=False,
49
+ rec_model_dir='rec_r45_abinet', det_model_dir='det_r50_vd_east_v2.0')
50
+
51
+ def perform_pad_ocr(image, conf=0.20, iou=0.91, imgsz=800):
52
+ if os.path.exists('cropped'):
53
+ shutil.rmtree('cropped/')
54
+
55
+ # make detection
56
+ results = model.predict(image, conf=conf, iou=iou, imgsz=imgsz)
57
+ ocr_result_path = "OCR_RESULTS"
58
+ image_fname = "uploaded_image"
59
+ chat1 = []
60
+
61
+ timestamp_formats = ['%I.%M %p', '%I %p', '%I.%M%p', '%I%M %p', '%I %p', '%I %M %p',
62
+ '%I:%M %p', '%I:%M:%S %p', '%H:%M', '%H.%M', '%H %M', '%H:%M:%S', '%H.%M.%S',
63
+ '%H %M %S', '%H:%M:%S.%f', '%H.%M.%S.%f', '%H %M %S %f']
64
+
65
+ for j in results:
66
+ os.makedirs(ocr_result_path, exist_ok=True)
67
+ boxes = j.boxes.cpu().numpy()
68
+ j.save(filename=f'{ocr_result_path}/{image_fname}_result.jpg') # save to disk
69
+
70
+ j.save_crop('cropped')
71
+
72
+ message_boxes = [box for box in boxes if j.names[int(box.cls[0])] in ['sender_message', 'receiver_message']]
73
+ replied_boxes = [box for box in boxes if j.names[int(box.cls[0])] in ['replied_to_message']]
74
+ message_boxes.sort(key=lambda box: box.xyxy[0][1])
75
+ replied_boxes.sort(key=lambda box: box.xyxy[0][1])
76
+ emoji_boxes = [box for box in boxes if j.names[int(box.cls[0])] == 'emoji']
77
+ timestamp_boxes = [box for box in boxes if j.names[int(box.cls[0])] == 'timestamp']
78
+
79
+ for msg_box, t_box in zip(message_boxes, timestamp_boxes):
80
+ message_type = j.names[int(msg_box.cls[0])]
81
+ msg_r = msg_box.xyxy[0].astype(int)
82
+ roi = (cv2.imread(image, 0))[msg_r[1]:msg_r[3], msg_r[0]:msg_r[2]]
83
+ ocr_res = ocr_model.ocr(roi, cls=True)[0]
84
+ ocr_text3 = " "
85
+ if ocr_res is not None:
86
+ for i in ocr_res:
87
+ txt = i[-1][0]
88
+ txt = txt.replace('\n', ' ')
89
+ txt = txt.strip()
90
+ ocr_text3 += f"{txt} "
91
+
92
+ t_i = cv2.imread(image)
93
+ fin_oc = []
94
+
95
+ ocr_text1 = " "
96
+
97
+ for i,emoji_box in enumerate(emoji_boxes):
98
+ emoji_r = emoji_box.xyxy[0].astype(int)
99
+ kernel = np.ones((8,8),np.float32)/15
100
+
101
+ t_i[emoji_r[1]:emoji_r[3], emoji_r[0]:emoji_r[2]] = cv2.filter2D(t_i[emoji_r[1]:emoji_r[3], emoji_r[0]:emoji_r[2]], -1, kernel)
102
+
103
+ center_x = int((emoji_r[0] + emoji_r[2]) / 2)
104
+ center_y = int((emoji_r[1] + emoji_r[3]) / 2)
105
+
106
+ text_size = cv2.getTextSize("%", cv2.FONT_HERSHEY_SIMPLEX, 1, 2)[0]
107
+ text_x = center_x - text_size[0] // 2
108
+ text_y = center_y + text_size[1] // 2
109
+
110
+ cv2.putText(t_i, "% ", (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0) , 2)
111
+ cv2.imwrite('t_i.jpg', t_i)
112
+
113
+ roi1 = (t_i[msg_r[1]:msg_r[3], msg_r[0]:msg_r[2]])
114
+ ocr_res1 = ocr_model.ocr(roi1, cls=True)[0]
115
+ new_oc = []
116
+ if ocr_res1 is not None:
117
+ for i in ocr_res1:
118
+ txt1 = i[-1][0]
119
+ txt1 = txt1.replace('\n', ' ')
120
+ txt1 = txt1.strip()
121
+ ocr_text1 += f"{txt1} "
122
+ new_oc.append(txt1)
123
+ fin_oc.append(new_oc)
124
+
125
+ ocr_text2 = " ".join(fin_oc[-1])
126
+ ocr_text3 = ocr_text2
127
+ ocr_text3 = re.sub(r'(EA|EF|EL|EM|EI|EN|EV|El|FM|FN|FI|FA|FE|FV|M E|E M|EE|MM|NM|MN|MI|NN|IM|M1|1M|MW|AN|SN|Q|%)+', '$$ ', ocr_text3)
128
+
129
+ rep_tt = ""
130
+ for replied_box in replied_boxes:
131
+ rep_r = replied_box.xyxy[0].astype(int)
132
+ if (msg_r[0] <= rep_r[0] and rep_r[2] <= msg_r[2]) and (msg_r[1] <= rep_r[1] and msg_r[3] <= rep_r[3]):
133
+ rep_roi = cv2.imread(image, 0)[rep_r[1]:rep_r[3], rep_r[0]:rep_r[2]]
134
+ rep_ocr_res = ocr_model.ocr(rep_roi, cls=True)[0]
135
+ if rep_ocr_res is not None:
136
+ for i in rep_ocr_res:
137
+ txt = i[-1][0]
138
+ rep_tt += f"{txt} "
139
+
140
+ try:
141
+ for time_box in timestamp_boxes:
142
+ time_r = time_box.xyxy[0].astype(int)
143
+ if (msg_r[0] <= time_r[0] and time_r[2] <= msg_r[2]) and (msg_r[1] <= time_r[1] and time_r[3] <= msg_r[3]):
144
+ ts_r = time_r
145
+ ts_roi = cv2.imread(image)[ts_r[1]:ts_r[3], ts_r[0]:ts_r[2]]
146
+ ts_ocr_res = ocr_model.ocr(ts_roi, cls=True)[0]
147
+ ts_text = ""
148
+ if ts_ocr_res is not None:
149
+ for i in ts_ocr_res:
150
+ txt = i[-1][0]
151
+ ts_text += txt
152
+ timestamp = txt
153
+ if timestamp is None:
154
+ match = re.search(r'\b\d{1,2}:\d{2}\s?(?:AM|PM)\b', ocr_text3)
155
+ timestamp = match.group() if match else None
156
+
157
+ ocr_text3 = ocr_text3.replace(timestamp, ' ') if timestamp else ocr_text3
158
+ ocr_text3 = re.sub(r'\b\d{1,2}:\d{2}\s?(?:AM|PM)\b', '', ocr_text3)
159
+ ocr_text3 = ocr_text1.replace(rep_tt, '') if rep_tt else ocr_text3
160
+ dictt1 = {"type": message_type, "text": ocr_text3.strip(), 'timestamp': timestamp}
161
+ chat1.append(dictt1)
162
+ except:
163
+ pass
164
+
165
+ json.dump(chat1, open(f"{ocr_result_path}/{image_fname}_result1.json", 'w'))
166
+ return chat1
167
+
168
+
169
+ def get_deep_message_extractor_ans(image_path, q):
170
+
171
+ # Construct the prompt with memory
172
+ fin_q = json.dumps(q, indent=2)
173
+ with open(image_path, "rb") as image_file:
174
+ image_base64 = base64.b64encode(image_file.read()).decode('utf-8')
175
+ prompt = (
176
+ f"From the original image"
177
+ f"You are a deep message and emoji extractor assistant.\n"
178
+ f"Your task is to process and analyze messages extracted text \n{fin_q} \n: from that image."
179
+ f"Your main focus is just removing duplicate dictionary with the following condition: \n"
180
+ f"If more than two dicts shares same text: values, remove the second dict and keep the first one\n"
181
+ f"Ensure to remove duplicates in dictionary, before outputing final JSON\n"
182
+ f"Do not modify the extracted texts; more importantly do not remove the emoji placeholders $$\n"
183
+ f"Ensure your final output is a JSON just like you receive it.\n"
184
+ f"If there are no duplicates, return original JSON with no modification\n"
185
+ f"If there are duplicates, JUST REMOVE the later dictionary and return final JSON, *** DO NOT MODIFY THE TEXTS, EXPECIALLY THE emoji placeholders $$ ***\n"
186
+ f"If you find emoji in the image, but not in the original extracted text, also have it replace with the placeholder $$\n"
187
+ f"Finally; make sure all the $$ emoji placeholders are replaced with unicode emoji in the final text or JSON\n"
188
+ f"And if no placeholder found, check from the image and make sure unicode emoji are correctly placed in the texts if available"
189
+ )
190
+
191
+ # Add the conversation history from memory
192
+ history = memory.load_memory_variables({})
193
+ if history['history']:
194
+ prompt = f"{history['history']}\n\n{prompt}"
195
+
196
+ # Invoke the ChatGroq model
197
+ # response = chat_model.invoke([HumanMessage(content=prompt)]).content.strip()
198
+
199
+ response = chat_model.invoke([
200
+ HumanMessage(
201
+ content=[
202
+ {"type": "text", "text": prompt},
203
+ {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}},
204
+ ]
205
+ )
206
+ ])
207
+
208
+ # Save the current interaction to memory
209
+ memory.save_context({"input": q}, {"output": response.content.strip()})
210
+
211
+ return response.content.strip()
212
+
213
+
214
+ def extract_json_from_response(response_text):
215
+ """
216
+ Extracts the JSON portion from the LLM's response.
217
+ """
218
+ # Use regex to find the JSON object or array
219
+ json_match = re.search(r'\[\s*\{.*\}\s*\]|\{\s*.*\s*\}', response_text, re.DOTALL)
220
+ if json_match:
221
+ try:
222
+ # Parse the extracted JSON
223
+ json_data = json.loads(json_match.group())
224
+ return json_data
225
+ except json.JSONDecodeError:
226
+ print("Failed to parse extracted JSON.")
227
+ return {"error": "Invalid JSON format in the response."}
228
+ else:
229
+ return {"error": "No JSON found in the response."}
230
+
231
+ def gradio_interface(image, conf, iou, imgsz):
232
+ res = perform_pad_ocr(image, conf, iou, imgsz)
233
+ fin_res = get_deep_message_extractor_ans(image, res)
234
+ ex_res = extract_json_from_response(fin_res)
235
+ # print(res)
236
+ return ex_res
237
+
238
+ # Create the Gradio interface
239
+ iface = gr.Interface(
240
+ fn=gradio_interface,
241
+ inputs=[
242
+ gr.Image(type="filepath", label="Upload Image"),
243
+ gr.Slider(minimum=0.25, maximum=1.0, step=0.01, value=0.25, label="Confidence Threshold"),
244
+ gr.Slider(minimum=0.45, maximum=1.0, step=0.01, value=0.91, label="IOU Threshold"),
245
+ gr.Slider(minimum=800, maximum=1000, step=32, value=800, label="Image Size")
246
+ ],
247
+ outputs=gr.JSON(label="OCR Results"),
248
+ title="deep message extractor",
249
+ description="Upload an image and adjust the parameters to extract text using YOLOv8 and PaddleOCR."
250
+ )
251
+
252
+ # # Create the FastAPI app
253
+ # app = FastAPI()
254
+
255
+ # # Add CORS middleware to allow all origins
256
+ # app.add_middleware(
257
+ # CORSMiddleware,
258
+ # allow_origins=["*"],
259
+ # allow_credentials=True,
260
+ # allow_methods=["*"],
261
+ # allow_headers=["*"],
262
+ # )
263
+
264
+ # # Mount the Gradio app
265
+ # app = gr.mount_gradio_app(app, iface, path="/")
266
+
267
+ # # Run the FastAPI app
268
+ # if __name__ == "__main__":
269
+ # import uvicorn
270
+ # uvicorn.run(app, host="0.0.0.0", port=80)
271
+
272
+ iface.launch(favicon_path='smile.png')
smile.png ADDED

Git LFS Details

  • SHA256: 06e7dffb4b866d20e978c6e51d26d829a81944d0cef9ccfe101b25937d324824
  • Pointer size: 131 Bytes
  • Size of remote file: 324 kB
test_images/ss_test1.png ADDED

Git LFS Details

  • SHA256: da2630a2b6a4f5071d61a00d08aca9d3e2ee14078a47d17a68eb34949e42eb5b
  • Pointer size: 131 Bytes
  • Size of remote file: 428 kB
test_images/ss_test2.png ADDED

Git LFS Details

  • SHA256: 8de555bad8ac0a991e388ef14f0e5f6678b0b9520d41baf603ea098a06352ffd
  • Pointer size: 131 Bytes
  • Size of remote file: 398 kB
test_images/ss_test3.png ADDED

Git LFS Details

  • SHA256: 53d69c3fefcc05bc129d38223f3b1a0bbc858e207cec17cfe63a82814258236c
  • Pointer size: 131 Bytes
  • Size of remote file: 360 kB
test_images/ss_test4.png ADDED

Git LFS Details

  • SHA256: 50e8cc054c88cb652d85fb79d8dbf0ef7e496da7ad6fcb69f54ca696713f7df8
  • Pointer size: 131 Bytes
  • Size of remote file: 461 kB
yolo_models/readme.md ADDED
@@ -0,0 +1 @@
 
 
1
+ Place yolo model here!.
yolo_models/yolov8l_imgsz800_map85_best.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd138b803101febb1c12375dcb14971c4a5665a72f3956dbb8d41287eba8e617
3
+ size 87723113