m8chaa commited on
Commit
7544e55
·
1 Parent(s): 27e7818

latest update

Browse files
Files changed (4) hide show
  1. .gitignore +7 -0
  2. app.py +584 -80
  3. books.json +31 -31
  4. requirements.txt +58 -5
.gitignore ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ /fastapi-tutorial
2
+ /env
3
+ __pycache__/
4
+ account_service.py
5
+ client_secret_512626967843-jptqk82outmq2nq3uk1pjb4lv2079jcv.apps.googleusercontent.com.json
6
+ resheet-uhd5gs-firebase-adminsdk-m49mq-c85cbeba8a.json
7
+ yungsoogi-b32c9aaed0a4.json
app.py CHANGED
@@ -1,86 +1,590 @@
1
- from fastapi import FastAPI, HTTPException, UploadFile, File
2
- from fastapi.responses import JSONResponse
 
 
 
 
3
  import numpy as np
4
  import cv2
5
- import io
6
  import base64
7
- from mangum import Mangum
 
 
 
 
 
 
 
8
 
9
  app = FastAPI()
10
 
11
- # Including necessary functions (such as angle_between_vectors_degrees, calculate_angles, and order_points) here
12
- def order_points(pts):
13
- rect = np.zeros((4, 2), dtype="float32")
14
- s = pts.sum(axis=1)
15
- rect[0] = pts[np.argmin(s)]
16
- rect[2] = pts[np.argmax(s)]
17
- diff = np.diff(pts, axis=1)
18
- rect[1] = pts[np.argmin(diff)]
19
- rect[3] = pts[np.argmax(diff)]
20
- return rect
21
- # Function to calculate angle
22
- def angle_between_vectors_degrees(u, v):
23
- """Returns the angle in degrees between vectors 'u' and 'v'."""
24
- cos_theta = np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))
25
- angle = np.arccos(np.clip(cos_theta, -1.0, 1.0))
26
- return np.degrees(angle)
27
-
28
- # Function to calculate angles of a contour
29
- def calculate_angles(contour):
30
- angles = []
31
- # Ensure the contour is of size (4,1,2)
32
- contour = contour.reshape((4, 2))
33
- # Loop through each corner
34
- for i in range(4):
35
- # Calculate vectors
36
- prev_vertex = contour[i - 1]
37
- current_vertex = contour[i]
38
- next_vertex = contour[(i + 1) % 4]
39
-
40
- vector1 = prev_vertex - current_vertex
41
- vector2 = next_vertex - current_vertex
42
-
43
- angle = angle_between_vectors_degrees(vector1, vector2)
44
- angles.append(angle)
45
- return angles
46
-
47
- @app.post("/process-receipts/")
48
- async def process_receipts(file: UploadFile = File(...)):
49
- if not file.content_type.startswith('image/'):
50
- raise HTTPException(status_code=400, detail=f"File '{file.filename}' is not an image.")
51
-
52
- try:
53
- contents = await file.read()
54
-
55
- # Convert the bytes to a NumPy array, then to an OpenCV image
56
- nparr = np.frombuffer(contents, np.uint8)
57
- cv_image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
58
-
59
- ret, bw_image = cv2.threshold(cv_image, 127, 255, cv2.THRESH_BINARY)
60
- blurred_bw_image = cv2.GaussianBlur(bw_image, (5, 5), 0)
61
- edges = cv2.Canny(blurred_bw_image, 75, 150)
62
- contours, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
63
-
64
- base64_images = []
65
- for contour in contours:
66
- perimeter = cv2.arcLength(contour, True)
67
- approximation = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
68
-
69
- if len(approximation) == 4:
70
- x, y, w, h = cv2.boundingRect(approximation) # Get bounding box of approximation
71
-
72
- # Proceed only if the bounding box is larger than 50px in both dimensions
73
- if w > 100 and h > 100:
74
- cropped_image = cv_image[y:y+h, x:x+w] # Cropped to bounding rectangle
75
-
76
- _, encoded_img = cv2.imencode('.PNG', cropped_image)
77
- base64_str = base64.b64encode(encoded_img).decode('utf-8')
78
- base64_images.append(base64_str)
79
-
80
- return JSONResponse(content={"receipt_images": base64_images})
81
-
82
- except Exception as e:
83
- raise HTTPException(status_code=500, detail=str(e))
84
-
85
- finally:
86
- await file.close()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from datetime import datetime
2
+ from uuid import uuid4
3
+ from fastapi import FastAPI, HTTPException, UploadFile, File, Request, BackgroundTasks, Form
4
+ from fastapi.middleware.cors import CORSMiddleware
5
+ from pydantic import BaseModel
6
+ from httpx import Timeout
7
  import numpy as np
8
  import cv2
 
9
  import base64
10
+ import httpx
11
+ import logging
12
+ import traceback
13
+ import boto3
14
+ import base64
15
+ import urllib.parse
16
+ from google.cloud import firestore
17
+ from google.oauth2 import service_account
18
 
19
  app = FastAPI()
20
 
21
+ app.add_middleware(
22
+ CORSMiddleware,
23
+ allow_origins=["*"], # Allows all origins
24
+ allow_credentials=True, # Allows credentials (such as cookies) to be sent with requests
25
+ allow_methods=["*"], # Allows all methods
26
+ allow_headers=["*"], # Allows all headers
27
+ )
28
+
29
+ timeout = Timeout(100.0, connect=600.0)
30
+
31
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
32
+ logging.info("Application starting up")
33
+
34
+ class ImageData(BaseModel):
35
+ sheetID: str
36
+ base64Image: str
37
+ fileName: str
38
+ mimeType: str
39
+
40
+ class UnprocessedImage(BaseModel):
41
+ sheetID: str
42
+ s3Url: str
43
+
44
+
45
+ # async def failure(sheetid):
46
+ # async with httpx.AsyncClient(follow_redirects=True) as client:
47
+ # response = await client.post(
48
+ # 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
49
+ # json={"sheetID": sheetid, "mode": "failed"}
50
+ # )
51
+ # if response.status_code != 200:
52
+ # logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
53
+ # raise HTTPException(status_code=500, detail="Error from external API.")
54
+
55
+ # async def download_image(url: str):
56
+ # async with httpx.AsyncClient(follow_redirects=True) as client:
57
+ # # Your existing code to stream the response
58
+ # # Use .stream() instead of .get() for streaming
59
+ # async with client.stream("GET", url) as resp:
60
+ # if resp.status_code == 200:
61
+ # content = b''
62
+ # async for chunk in resp.aiter_bytes():
63
+ # content += chunk
64
+ # return content
65
+ # else:
66
+ # return None
67
+ #
68
+ # async def save_image(image_data, image):
69
+ # s3_client = boto3.client('s3')
70
+
71
+ # bucket_name = 're-sheet'
72
+ # object_name = image_data['fileName']
73
+
74
+ # response = s3_client.put_object(
75
+ # Bucket=bucket_name,
76
+ # Key=object_name,
77
+ # Body=image,
78
+ # ContentType=image_data['mimeType']
79
+ # )
80
+
81
+ # status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
82
+ # if status == 200:
83
+ # message = "File uploaded successfully to S3"
84
+ # image_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
85
+ # body = {"queued":
86
+ # [{
87
+ # "s3Url": image_url,
88
+ # "sheetID": image_data['sheetID'],
89
+ # "type": "process-receipts"
90
+ # },],
91
+ # "mode": "queue",
92
+ # }
93
+
94
+ # async with httpx.AsyncClient(follow_redirects=True) as client:
95
+ # response = await client.post(
96
+ # 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
97
+ # json=body
98
+ # )
99
+ # logging.info(f"Response from Google Apps Script API: {response.status_code} - {response.text}")
100
+ # if response.status_code != 200:
101
+ # logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
102
+ # failure(image_data['sheetID'])
103
+ # raise HTTPException(status_code=500, detail="Error from external API.")
104
+
105
+ @app.post("/redirect")
106
+ async def redirect_to_google_apps_script(request: Request):
107
+ # Get the body of the incoming request
108
+ body = await request.body()
109
+
110
+ async with httpx.AsyncClient(timeout=300.0, follow_redirects=True) as client: # Follow redirects
111
+ response = await client.post(
112
+ 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
113
+ data=body
114
+ )
115
+ if response.status_code != 200:
116
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
117
+ raise HTTPException(status_code=500, detail="Error from external API.")
118
+ return response.json()
119
+
120
+ # @app.post("/save-to-s3")
121
+ # async def save_to_s3(request: Request, background_tasks: BackgroundTasks):
122
+ # image_data = await request.json()
123
+
124
+ # image = await download_image(image_data['image_url'])
125
+ # if image is None:
126
+ # raise HTTPException(status_code=500, detail="Cannot download image from google drive.")
127
+
128
+ # background_tasks.add_task(save_image, image_data, image)
129
+ # return {"message": "Image download started. Processing in the background."}
130
+
131
+
132
+ # @app.post("/upload-and-save")
133
+ # async def upload_and_save(background_tasks: BackgroundTasks, image: UploadFile = File(...), sheet_url: str = Form(...)):
134
+ # image_contents = await image.read()
135
+ # sheet_id = sheet_url.split('/')[-2]
136
+ # image_data = {
137
+ # 'sheetID': sheet_id,
138
+ # 'fileName': image.filename,
139
+ # 'mimeType': image.content_type
140
+ # }
141
+
142
+ # background_tasks.add_task(save_image, image_data, image_contents)
143
+ # return {"message": "Image upload started. Processing in the background."}
144
+
145
+
146
+ # def order_points(pts):
147
+ # rect = np.zeros((4, 2), dtype="float32")
148
+ # s = pts.sum(axis=1)
149
+ # rect[0] = pts[np.argmin(s)]
150
+ # rect[2] = pts[np.argmax(s)]
151
+ # diff = np.diff(pts, axis=1)
152
+ # rect[1] = pts[np.argmin(diff)]
153
+ # rect[3] = pts[np.argmax(diff)]
154
+ # return rect
155
+
156
+ # def angle_between_vectors_degrees(u, v):
157
+ # """Returns the angle in degrees between vectors 'u' and 'v'."""
158
+ # cos_theta = np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))
159
+ # angle = np.arccos(np.clip(cos_theta, -1.0, 1.0))
160
+ # return np.degrees(angle)
161
+
162
+ # def calculate_angles(contour):
163
+ # angles = []
164
+ # contour = contour.reshape((4, 2))
165
+ # for i in range(4):
166
+ # prev_vertex = contour[i - 1]
167
+ # current_vertex = contour[i]
168
+ # next_vertex = contour[(i + 1) % 4]
169
+
170
+ # vector1 = prev_vertex - current_vertex
171
+ # vector2 = next_vertex - current_vertex
172
+
173
+ # angle = angle_between_vectors_degrees(vector1, vector2)
174
+ # angles.append(angle)
175
+ # return angles
176
+
177
+ # def detectReceipt(masked_image, original_image):
178
+ # original_aspect_ratio = original_image.shape[1] / original_image.shape[0]
179
+ # masked_aspect_ratio = masked_image.shape[1] / masked_image.shape[0]
180
+
181
+ # if abs(original_aspect_ratio - masked_aspect_ratio) > 0.1: # Threshold can be adjusted
182
+ # # Rotate the masked image 90 degrees clockwise
183
+ # masked_image = cv2.rotate(masked_image, cv2.ROTATE_90_CLOCKWISE)
184
+
185
+ # original_image_shape = original_image.shape[:2]
186
+
187
+ # resized_masked_image = cv2.resize(masked_image, (original_image_shape[1], original_image_shape[0]))
188
+
189
+
190
+
191
+ # blurred_masked_image = cv2.GaussianBlur(resized_masked_image, (5, 5), 0)
192
+ # _, thresh_masked_image = cv2.threshold(blurred_masked_image, 175, 255, cv2.THRESH_BINARY)
193
+ # edges = cv2.Canny(thresh_masked_image, 75, 150)
194
+ # contours, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
195
+
196
+ # base64_images = []
197
+ # padding = 20
198
+ # for contour in contours:
199
+ # perimeter = cv2.arcLength(contour, True)
200
+ # approximation = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
201
+ # x, y, w, h = cv2.boundingRect(approximation)
202
+
203
+ # # Add padding to the bounding box coordinates while ensuring they are within the image boundaries
204
+ # x_pad = max(x - padding, 0)
205
+ # y_pad = max(y - padding, 0)
206
+ # w_pad = min(w + 2 * padding, original_image.shape[1] - x_pad)
207
+ # h_pad = min(h + 2 * padding, original_image.shape[0] - y_pad)
208
+
209
+ # # logging.info(f"Bounding box: x={x_pad}, y={y_pad}, w={w_pad}, h={h_pad}")
210
+
211
+ # # Check if the area of the bounding rectangle is greater than 50px x 50px
212
+ # if w_pad > 150 or h_pad > 150:
213
+ # cropped_image = original_image[y_pad:y_pad + h_pad, x_pad:x_pad + w_pad]
214
+ # _, img_encoded = cv2.imencode('.png', cropped_image)
215
+ # base64_str = base64.b64encode(img_encoded).decode('utf-8')
216
+ # base64_images.append(base64_str)
217
+
218
+ # return base64_images
219
+
220
+
221
+ # # process images as background tasks
222
+ # async def process_image(original_contents, s3_client, object_key, bucket_name, image, original_image):
223
+
224
+ # logging.info("Preparing request to Dezgo API")
225
+
226
+ # headers = {
227
+ # 'X-Dezgo-Key': 'DEZGO-AA84DEB300562DA089E56CEA6081AE7DB601EF2D3EE4C693910E1C25E59A4FF8023D4932'
228
+ # }
229
+ # files = {
230
+ # 'image': ('filename', original_contents),
231
+ # 'mode': 'mask',
232
+ # }
233
+
234
+ # logging.info("Sending request to Dezgo API")
235
+ # async with httpx.AsyncClient(timeout=timeout) as client:
236
+ # response = await client.post('https://api.dezgo.com/remove-background', headers=headers, files=files)
237
+ # if response.status_code != 200:
238
+ # logging.error(f"Unexpected response from Dezgo API: {response.status_code} - {response.text}")
239
+ # failure(image['sheetID'])
240
+ # raise HTTPException(status_code=500, detail="Error from external API.")
241
+ # masked_image_data = response.content
242
+
243
+ # if not masked_image_data:
244
+ # logging.error("Received empty masked image data from Dezgo API.")
245
+ # failure(image['sheetID'])
246
+ # raise HTTPException(status_code=500, detail="Received empty data from external API.")
247
+
248
+ # nparr_masked = np.frombuffer(masked_image_data, np.uint8)
249
+ # if nparr_masked.size == 0:
250
+ # logging.error("Failed to convert API response to numpy array.")
251
+ # failure(image['sheetID'])
252
+ # raise HTTPException(status_code=500, detail="Error processing API response data.")
253
+
254
+ # masked_image = cv2.imdecode(nparr_masked, cv2.IMREAD_COLOR)
255
+ # if masked_image is None:
256
+ # logging.error("Failed to decode image data.")
257
+ # failure(image['sheetID'])
258
+ # raise HTTPException(status_code=500, detail="Error decoding image data.")
259
+
260
+ # # Call the detectReceipt function to process the image
261
+ # base64_images = detectReceipt(masked_image, original_image)
262
+
263
+
264
+
265
+ # queued = []
266
+ # for i, base64_str in enumerate(base64_images, start=1):
267
+ # image_bytes = base64.b64decode(base64_str)
268
+ # object_name = f"{object_key}_processed_{i}.png"
269
+
270
+ # # Upload the processed image to S3
271
+ # response = s3_client.put_object(
272
+ # Bucket=bucket_name,
273
+ # Key=object_name,
274
+ # Body=image_bytes,
275
+ # ContentType='image/png'
276
+ # )
277
+ # print('saving to s3')
278
+
279
+ # status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
280
+ # if status == 200:
281
+ # url_encoded_object_name = urllib.parse.quote(object_name)
282
+ # processed_image_url = f"https://{bucket_name}.s3.amazonaws.com/{url_encoded_object_name}"
283
+ # body = {
284
+ # "s3Url": processed_image_url,
285
+ # "sheetID": image['sheetID'],
286
+ # "mode": "queue",
287
+ # "type": "read-receipt"
288
+ # }
289
+ # queued.append(body)
290
+ # else:
291
+ # logging.error(f"Failed to upload processed image to S3: {object_name}")
292
+ # body = {"queued": queued, "mode": "queue", }
293
+ # async with httpx.AsyncClient(follow_redirects=True) as client:
294
+ # response = await client.post(
295
+ # 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec',
296
+ # json=body
297
+ # )
298
+ # if response.status_code != 200:
299
+ # failure(image['sheetID'])
300
+ # logging.error(
301
+ # f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
302
+ # raise HTTPException(status_code=500, detail="Error from external API.")
303
+
304
+ # @app.post("/process-receipts")
305
+ # async def process_receipts(request: Request, background_tasks: BackgroundTasks):
306
+ # image = await request.json()
307
+ # logging.info(f"Received file: {image['s3Url']}")
308
+
309
+ # s3_client = boto3.client('s3')
310
+ # bucket_name = image["s3Url"].split('/')[2].split('.')[0] # Extract bucket name from the URL
311
+ # object_key = '/'.join(image["s3Url"].split('/')[3:]) # Extract object key from the URL
312
+
313
+ # try:
314
+ # # Download the image from S3
315
+ # s3_response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
316
+ # original_contents = s3_response['Body'].read() # gets the image as bytes
317
+
318
+ # logging.info("Decoding the original image for processing")
319
+ # nparr_original = np.frombuffer(original_contents, np.uint8)
320
+ # original_image = cv2.imdecode(nparr_original, cv2.IMREAD_COLOR)
321
+
322
+ # if original_image is None:
323
+ # logging.error("Failed to decode the uploaded image.")
324
+ # raise HTTPException(status_code=500, detail="Failed to decode the uploaded image.")
325
+
326
+ # background_tasks.add_task(process_image, original_contents, s3_client, object_key, bucket_name, image, original_image)
327
+
328
+ # return {"message": "image queued for processing"}
329
+
330
+ # except Exception as e:
331
+ # error_msg = f"Error during processing: {e}\n{traceback.format_exc()}"
332
+ # failure(image['sheetID'])
333
+ # logging.error(error_msg)
334
+ # raise HTTPException(status_code=500, detail=error_msg)
335
+
336
+ # finally:
337
+ # logging.info("File processing completed")
338
+
339
+
340
+
341
+
342
+
343
+ # else:
344
+ # raise HTTPException(status_code=400, detail="Email address not provided.")
345
+ appsScriptsURL = 'https://script.google.com/macros/s/AKfycbw9_31pMwQVhJPezJ9qlMRCAamQKRREnun-tfT_TnS5TOwOVcfdgU1Y9xTpAv6bZuiKOA/exec'
346
+ dezgo_api_key = 'DEZGO-AA84DEB300562DA089E56CEA6081AE7DB601EF2D3EE4C693910E1C25E59A4FF8023D4932'
347
+
348
+ dezgo_headers = {
349
+ 'X-Dezgo-Key': dezgo_api_key,
350
+ }
351
+
352
+ # update the google sheet with failure
353
+ async def record_failure(s3_original_url, failed_task, sheetID):
354
+ body = {
355
+ "s3Url": s3_original_url,
356
+ "mode": "failed",
357
+ "message": failed_task,
358
+ "sheetID": sheetID,
359
+ }
360
+ async with httpx.AsyncClient(follow_redirects=True) as client:
361
+ response = await client.post(appsScriptsURL, json=body)
362
+ if response.status_code != 200:
363
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
364
+ raise HTTPException(status_code=500, detail="Error from external API.")
365
+
366
+ async def post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url):
367
+ async with httpx.AsyncClient(follow_redirects=True) as client:
368
+ response = await client.post(appsScriptsURL, json=body)
369
+ if response.status_code != 200:
370
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
371
+ await record_failure(s3_orginal_url, 'Failed to queue processed image.', body.sheetID)
372
+
373
+ async def background_worker(sheeturl, image_contents, filename, content_type, appsScriptsURL):
374
+ sheetID = sheeturl.split('/')[-2]
375
+ image_data = {
376
+ 'sheetID': sheetID,
377
+ 'fileName': filename,
378
+ 'mimeType': content_type
379
+ }
380
+
381
+ s3_client = boto3.client('s3')
382
+ bucket_name = 're-sheet'
383
+ object_name = sheetID + '/' + image_data['fileName']
384
+
385
+ s3response = s3_client.put_object(
386
+ Bucket=bucket_name,
387
+ Key=object_name,
388
+ Body=image_contents,
389
+ ContentType=image_data['mimeType']
390
+ )
391
+ cut_images = []
392
+ s3_orginal_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
393
+ masked_image = await detect_receipts(image_contents, s3_orginal_url, sheetID)
394
+ await cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID)
395
+
396
+ queued = []
397
+ # save the processed image(s) to s3
398
+ if not cut_images:
399
+ await record_failure(s3_orginal_url, 'No valid receipts found.', sheetID)
400
+ return {"message": "No valid receipts found."}
401
+ else:
402
+ for i, base64_str in enumerate(cut_images, start=1):
403
+ image_bytes = base64.b64decode(base64_str)
404
+ object_name = f"{object_name}_processed_{i}.png"
405
+ response = s3_client.put_object(
406
+ Bucket=bucket_name,
407
+ Key=object_name,
408
+ Body=image_bytes,
409
+ ContentType='image/png'
410
+ )
411
+ status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
412
+ if status != 200:
413
+ await record_failure(s3_orginal_url, 'Failed to upload processed image to S3.', sheetID)
414
+ return {"message": "Failed to upload processed image to S3."}
415
+ # otherwise put it as queued
416
+ object_s3_url = f"https://{bucket_name}.s3.amazonaws.com/{object_name}"
417
+ queued.append(object_s3_url)
418
+ # queue the image to google apps script web app
419
+ body ={
420
+ "queued": queued,
421
+ "mode": "mobile",
422
+ "sheetID": sheetID,
423
+ }
424
+
425
+ print(body)
426
+ logging.info(body)
427
+ await post_to_google_apps_script(appsScriptsURL, body, s3_orginal_url)
428
+ # receive a sheetID and an image as multipart form data
429
+ @app.post('/mobile-process-receipts')
430
+ async def mobile_process_receipts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), image: UploadFile = File(...)):
431
+ image_contents = await image.read()
432
+ uuid = str(uuid4())
433
+ timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
434
+ message = "영수기 AI가 데이터 분석 중입니다. 잠시만 기다려주세요."
435
+ data = [timestamp, uuid, message]
436
+ body = {
437
+ "sheetID": sheeturl.split('/')[-2],
438
+ "data": data,
439
+ "mode": "alert-user"
440
+ }
441
+ async with httpx.AsyncClient(follow_redirects=True) as client:
442
+ response = await client.post(appsScriptsURL, json=body)
443
+ if response.status_code != 200:
444
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
445
+ raise HTTPException(status_code=500, detail="Error from external API.")
446
+
447
+ background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, image.content_type, appsScriptsURL)
448
+
449
+ return {"message": "Image processing completed."}
450
+
451
+ @app.post('/process-receipts')
452
+ async def mobile_process_receipts(background_tasks: BackgroundTasks, request: Request):
453
+ data = await request.json()
454
+ image_contents = base64.b64decode(data['base64Image'])
455
+ content_type = data['mimeType']
456
+ sheeturl = data['sheetID']
457
+ uuid = str(uuid4())
458
+ timestamp = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
459
+ message = "영수기 AI가 데이터 분석 중입니다. 잠시만 기다려주세요."
460
+ data = [timestamp, uuid, message]
461
+ body = {
462
+ "sheetID": sheeturl.split('/')[-2],
463
+ "data": data,
464
+ "mode": "alert-user"
465
+ }
466
+ async with httpx.AsyncClient(follow_redirects=True) as client:
467
+ response = await client.post(appsScriptsURL, json=body)
468
+ if response.status_code != 200:
469
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
470
+ raise HTTPException(status_code=500, detail="Error from external API.")
471
+
472
+ background_tasks.add_task(background_worker, sheeturl, image_contents, uuid, content_type, appsScriptsURL)
473
+
474
+ return {"message": "Image processing completed."}
475
+
476
+ # send the image to dezgo
477
+ async def detect_receipts(image_contents, s3_orginal_url, sheetID):
478
+ files = {
479
+ 'image': ('filename', image_contents),
480
+ 'mode': 'mask',
481
+ }
482
+ logging.info("Sending image to Dezgo API")
483
+ async with httpx.AsyncClient(timeout=timeout) as client:
484
+ response = await client.post('https://api.dezgo.com/remove-background', headers=dezgo_headers, files=files)
485
+ # if successful, pass the masked image to cutting_image function
486
+ if response.status_code == 200:
487
+ masked_image = response.content
488
+ return masked_image
489
+ else:
490
+ logging.error(f"Unexpected response from Dezgo API: {response.status_code} - {response.text}")
491
+ await record_failure(s3_orginal_url, 'Dezgo API failed.', sheetID)
492
+ raise HTTPException(status_code=500, detail="Error from external API.")
493
+
494
+ # receive the masked image and process it
495
+ async def cutting_image(masked_image, image_contents, cut_images, s3_orginal_url, sheetID):
496
+ # Convert the masked_image (which is in bytes format) to a numpy array.
497
+ # The dtype of the array is set to np.uint8 because images are usually 8-bit per channel.
498
+ nparr_masked = np.frombuffer(masked_image, np.uint8)
499
+
500
+ # Check if the size of the numpy array is 0, which means that the conversion failed.
501
+ if nparr_masked.size == 0:
502
+ # Log an error message indicating that the conversion failed.
503
+ logging.error("Failed to convert API response to numpy array.")
504
+ # Record the reason for the failure.
505
+ record_failure(s3_orginal_url, 'failed to convert API response to numpy array.', sheetID)
506
+ # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
507
+ raise HTTPException(status_code=500, detail="Error processing API response data.")
508
+
509
+ # Decode the numpy array to an image using OpenCV. The flag cv2.IMREAD_COLOR loads the image in color.
510
+ masked_image = cv2.imdecode(nparr_masked, cv2.IMREAD_COLOR)
511
+
512
+ # Check if the decoding failed, which would result in masked_image being None.
513
+ if masked_image is None:
514
+ # Log an error message indicating that the decoding failed.
515
+ logging.error("Failed to decode image data.")
516
+ # Record the reason for the failure.
517
+ record_failure(s3_orginal_url, 'failed to decode image data.', sheetID)
518
+ # Raise an HTTPException with a 500 status code (Internal Server Error) and a detail message.
519
+ raise HTTPException(status_code=500, detail="Error decoding image data.")
520
+
521
+ nparr_original = np.frombuffer(image_contents, np.uint8)
522
+ original_image = cv2.imdecode(nparr_original, cv2.IMREAD_COLOR)
523
+
524
+ original_aspect_ratio = original_image.shape[1] / original_image.shape[0]
525
+ masked_aspect_ratio = masked_image.shape[1] / masked_image.shape[0]
526
+
527
+ # if the image has been rotated by dezgo, rotate it back
528
+ if abs(original_aspect_ratio - masked_aspect_ratio) > 0.1:
529
+ # Rotate the masked image 90 degrees clockwise
530
+ masked_image = cv2.rotate(masked_image, cv2.ROTATE_90_CLOCKWISE)
531
+
532
+ original_image_shape = original_image.shape[:2]
533
+
534
+ resized_masked_image = cv2.resize(masked_image, (original_image_shape[1], original_image_shape[0]))
535
+
536
+
537
+
538
+ blurred_masked_image = cv2.GaussianBlur(resized_masked_image, (5, 5), 0)
539
+ _, thresh_masked_image = cv2.threshold(blurred_masked_image, 175, 255, cv2.THRESH_BINARY)
540
+ edges = cv2.Canny(thresh_masked_image, 75, 150)
541
+ contours, hierarchy = cv2.findContours(edges.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
542
+
543
+ padding = 20
544
+ for contour in contours:
545
+ perimeter = cv2.arcLength(contour, True)
546
+ approximation = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
547
+ x, y, w, h = cv2.boundingRect(approximation)
548
+
549
+ # Add padding to the bounding box coordinates while ensuring they are within the image boundaries
550
+ x_pad = max(x - padding, 0)
551
+ y_pad = max(y - padding, 0)
552
+ w_pad = min(w + 2 * padding, original_image.shape[1] - x_pad)
553
+ h_pad = min(h + 2 * padding, original_image.shape[0] - y_pad)
554
+
555
+ # logging.info(f"Bounding box: x={x_pad}, y={y_pad}, w={w_pad}, h={h_pad}")
556
+
557
+ # Check if the area of the bounding rectangle is greater than 50px x 50px
558
+ if w_pad > 150 or h_pad > 150:
559
+ cropped_image = original_image[y_pad:y_pad + h_pad, x_pad:x_pad + w_pad]
560
+ _, img_encoded = cv2.imencode('.png', cropped_image)
561
+ base64_str = base64.b64encode(img_encoded).decode('utf-8')
562
+ cut_images.append(base64_str)
563
+ # -> failed or empty: send a response of image and sheetid to the google sheet
564
+ # save the processed image(s) to s3
565
+ # -> failed or empty: send a response of image and sheetid to the google sheet
566
+ # use google ocr to extract text from the image
567
+ # -> failed or empty: send a response of image and sheetid to the google sheet
568
+ # send the text to Openai
569
+ # -> failed or empty: send a response of image and sheetid to the google sheet
570
+ # send the response to the google sheet
571
+
572
+ @app.post('/test')
573
+ async def mobile_process_receipts(background_tasks: BackgroundTasks, sheeturl: str = Form(...), image: UploadFile = File(...)):
574
+ image_contents = await image.read()
575
+ uuid = str(uuid4())
576
+ timestamp = str(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
577
+ message = "영수기가 데이터 분석 중입니다. 잠시만 기다려주세요."
578
+ data = [timestamp, uuid, message]
579
+ body = {
580
+ "sheetID": sheeturl.split('/')[-2],
581
+ "data": data,
582
+ "mode": "alert-user"
583
+ }
584
+ async with httpx.AsyncClient(follow_redirects=True) as client:
585
+ response = await client.post(appsScriptsURL, json=body)
586
+ if response.status_code != 200:
587
+ logging.error(f"Unexpected response from Google Apps Script API: {response.status_code} - {response.text}")
588
+ raise HTTPException(status_code=500, detail="Error from external API.")
589
+ background_tasks.add_task(background_worker, sheeturl, image_contents, image.filename, image.content_type, appsScriptsURL)
590
+ return {"message": "Image processing completed."}
books.json CHANGED
@@ -1,32 +1,32 @@
1
  [
2
- {
3
- "name": "Harry Potter",
4
- "genre": "fiction",
5
- "price": 15.0,
6
- "book_id": "e1b7e31916f64bd3badd98b9e34cc5c3"
7
- },
8
- {
9
- "name": "The Great Gatsby",
10
- "genre": "fiction",
11
- "price": 15.0,
12
- "book_id": "1759c5e1cf4b4e26b7dd41c7d2af7c33"
13
- },
14
- {
15
- "name": "Frankenstein",
16
- "genre": "fiction",
17
- "price": 10.0,
18
- "book_id": "f4b7f7f488334155831a9656ba373530"
19
- },
20
- {
21
- "name": "Atomic Habits",
22
- "genre": "non-fiction",
23
- "price": 30.0,
24
- "book_id": "6e72714965f94d0c9600277bb7e5ba99"
25
- },
26
- {
27
- "name": "A Wild Sheep Chase",
28
- "genre": "fiction",
29
- "price": 15.0,
30
- "book_id": "58bfb9a684724846bd7f69688c6f09c9"
31
- }
32
- ]
 
1
  [
2
+ {
3
+ "name": "Harry Potter",
4
+ "genre": "fiction",
5
+ "price": 15.0,
6
+ "book_id": "e1b7e31916f64bd3badd98b9e34cc5c3"
7
+ },
8
+ {
9
+ "name": "The Great Gatsby",
10
+ "genre": "fiction",
11
+ "price": 15.0,
12
+ "book_id": "1759c5e1cf4b4e26b7dd41c7d2af7c33"
13
+ },
14
+ {
15
+ "name": "Frankenstein",
16
+ "genre": "fiction",
17
+ "price": 10.0,
18
+ "book_id": "f4b7f7f488334155831a9656ba373530"
19
+ },
20
+ {
21
+ "name": "Atomic Habits",
22
+ "genre": "non-fiction",
23
+ "price": 30.0,
24
+ "book_id": "6e72714965f94d0c9600277bb7e5ba99"
25
+ },
26
+ {
27
+ "name": "A Wild Sheep Chase",
28
+ "genre": "fiction",
29
+ "price": 15.0,
30
+ "book_id": "58bfb9a684724846bd7f69688c6f09c9"
31
+ }
32
+ ]
requirements.txt CHANGED
@@ -1,5 +1,58 @@
1
- fastapi
2
- uvicorn
3
- mangum
4
- python-multipart
5
- opencv-python-headless
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ annotated-types==0.6.0
2
+ anyio==4.3.0
3
+ boto3==1.34.48
4
+ botocore==1.34.48
5
+ CacheControl==0.14.0
6
+ cachetools==5.3.3
7
+ certifi==2024.2.2
8
+ cffi==1.16.0
9
+ charset-normalizer==3.3.2
10
+ click==8.1.7
11
+ cryptography==42.0.5
12
+ fastapi==0.109.2
13
+ firebase-admin==6.5.0
14
+ google-api-core==2.18.0
15
+ google-api-python-client==2.123.0
16
+ google-auth==2.29.0
17
+ google-auth-httplib2==0.2.0
18
+ google-cloud-core==2.4.1
19
+ google-cloud-datastore==2.19.0
20
+ google-cloud-firestore==2.15.0
21
+ google-cloud-storage==2.16.0
22
+ google-crc32c==1.5.0
23
+ google-resumable-media==2.7.0
24
+ googleapis-common-protos==1.63.0
25
+ grpcio==1.62.1
26
+ grpcio-status==1.62.1
27
+ h11==0.14.0
28
+ httpcore==1.0.3
29
+ httplib2==0.22.0
30
+ httpx==0.26.0
31
+ idna==3.6
32
+ jmespath==1.0.1
33
+ mangum==0.17.0
34
+ msgpack==1.0.8
35
+ numpy==1.26.4
36
+ opencv-python-headless==4.9.0.80
37
+ pillow==10.2.0
38
+ proto-plus==1.23.0
39
+ protobuf==4.25.3
40
+ pyasn1==0.5.1
41
+ pyasn1-modules==0.3.0
42
+ pycparser==2.21
43
+ pydantic==2.6.1
44
+ pydantic_core==2.16.2
45
+ PyJWT==2.8.0
46
+ pyparsing==3.1.2
47
+ python-dateutil==2.8.2
48
+ python-multipart==0.0.9
49
+ requests==2.31.0
50
+ rsa==4.9
51
+ s3transfer==0.10.0
52
+ six==1.16.0
53
+ sniffio==1.3.0
54
+ starlette==0.36.3
55
+ typing_extensions==4.9.0
56
+ uritemplate==4.1.1
57
+ urllib3==2.0.7
58
+ uvicorn==0.27.1